Skip to content
Snippets Groups Projects
Commit 7a0bdba9 authored by Alex Wiens's avatar Alex Wiens
Browse files

prule.db,summary: Update summary

parent da2a23a8
No related branches found
No related tags found
No related merge requests found
......@@ -4,16 +4,32 @@ Common database functions for PRULE results database.
import datetime
import sqlite3
import prule.debug
import threading
import concurrent.futures
class ResultsDB:
def __init__(self, path):
def __init__(self, path, trace_sql=None, ro=False):
self.path = path
# check sqlite3.threadsafety to see if check_same_thread can be safely deactivated
if ro == True:
#self.con = sqlite3.connect("file:{}?mode=ro&immutable=1&nolock=1".format(path))
self.con = sqlite3.connect("file:{}?mode=ro&immutable=1".format(path))
else:
self.con = sqlite3.connect(path)
if trace_sql != None:
self.con.set_trace_callback(trace_sql)
#self.config()
self.rule_dict = None
def close(self):
if self.con != None:
self.con.close()
self.con = None
def config(self):
cur = self.con.cursor()
cur.execute("PRAGMA cache_size=10000;")
self.con.commit()
def db_check(self):
cur = self.con.cursor()
res = cur.execute("SELECT name FROM sqlite_master")
......@@ -365,3 +381,60 @@ rulename TEXT
res = cur.execute('SELECT COUNT(*) FROM jobs where evaluated=1 and cluster=?;', [cluster])
evaluated = res.fetchall()[0][0]
return (evaluated, rule_match, rule_eval, rule_error)
"""
# Use separate connection
mult = MultiDb(db_con)
future_result = mult.submitobj("all_results", db_con, prule.db.ResultsDB.db_get_all_results, columns=['COUNT(ccid)'], transform=False)
result = future_result.result()
# Use same connection
mult = MultiDb(db_con)
future_result = mult.submit("non_processed", db_con.db_get_all_results, columns=['COUNT(processed)'], conditions=["processed != 1"], transform=False)
result = future_result.result()
# Multiple queries
mult = MultiDb(db_con)
queries = {}
queries["one"] = mult.submit(...)
queries["two"] = mult.submit(...)
concurrent.futures.wait(queries.values())
one = queries["one"].result()
two = queries["two"].result()
# don't forget to close the pool
mult.shutdown()
"""
class MultiDb:
def __init__(self, db):
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, initializer=MultiDb.init_thread)
self.db = db
def init_thread():
pid = os.getpid()
ident = threading.get_native_id()
aff = None
if hasattr(os, "sched_getaffinity"):
aff = os.sched_getaffinity(0)
print(pid, ident, aff)
def measure(name, fun, args, kwargs):
with prule.debug.Timing(name, True):
return fun(*args, **kwargs)
def measureobj(name, obj, fun, args, kwargs):
with prule.debug.Timing(name, True):
return fun(obj, *args, **kwargs)
def submit(self, name, fun, *args, **kwargs):
#fut = self.executor.submit(fun, arg)
fut = self.executor.submit(MultiDb.measure, name, fun, args, kwargs)
return fut
def submitobj(self, name, obj, fun, *args, **kwargs):
#fut = self.executor.submit(fun, arg)
newdb = prule.db.ResultsDB(self.db.path, ro=True)
fut = self.executor.submit(MultiDb.measureobj, name, newdb, fun, args, kwargs)
return fut
def shutdown(self):
self.executor.shutdown(wait=False)
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment