diff --git a/README.md b/README.md index 8619b3a6..4b694640 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,8 @@ the same arguments can be set as volume flags, in addition to `d2d` and `d2t` fo `e2tsr` is probably always overkill, since `e2ds`/`e2dsa` would pick up any file modifications and cause `e2ts` to reindex those +the rescan button in the admin panel has no effect unless the volume has `-e2ds` or higher + ## metadata from audio files diff --git a/copyparty/__main__.py b/copyparty/__main__.py index c7014e0e..5eb12f92 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -249,6 +249,10 @@ def run_argparse(argv, formatter): ap.add_argument("--urlform", metavar="MODE", type=str, default="print,get", help="how to handle url-forms") ap.add_argument("--salt", type=str, default="hunter2", help="up2k file-hash salt") + ap2 = ap.add_argument_group('admin panel options') + ap2.add_argument("--no-rescan", action="store_true", help="disable ?scan (volume reindexing)") + ap2.add_argument("--no-stack", action="store_true", help="disable ?stack (list all stacks)") + ap2 = ap.add_argument_group('thumbnail options') ap2.add_argument("--no-thumb", action="store_true", help="disable all thumbnails") ap2.add_argument("--no-vthumb", action="store_true", help="disable video thumbnails") @@ -287,6 +291,7 @@ def run_argparse(argv, formatter): ap2.add_argument("--log-conn", action="store_true", help="print tcp-server msgs") ap2.add_argument("--no-sendfile", action="store_true", help="disable sendfile") ap2.add_argument("--no-scandir", action="store_true", help="disable scandir") + ap2.add_argument("--no-fastboot", action="store_true", help="wait for up2k indexing") ap2.add_argument("--ihead", metavar="HEADER", action='append', help="dump incoming header") ap2.add_argument("--lf-url", metavar="RE", type=str, default=r"^/\.cpr/|\?th=[wj]$", help="dont log URLs matching") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 31fe03df..82d6cd78 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -14,11 +14,12 @@ from .util import IMPLICATIONS, undot, Pebkac, fsdec, fsenc, statdir, nuprint class VFS(object): """single level in the virtual fs""" - def __init__(self, realpath, vpath, uread=[], uwrite=[], flags={}): + def __init__(self, realpath, vpath, uread=[], uwrite=[], uadm=[], flags={}): self.realpath = realpath # absolute path on host filesystem self.vpath = vpath # absolute path in the virtual filesystem self.uread = uread # users who can read this self.uwrite = uwrite # users who can write this + self.uadm = uadm # users who are regular admins self.flags = flags # config switches self.nodes = {} # child nodes self.all_vols = {vpath: self} # flattened recursive @@ -27,7 +28,7 @@ class VFS(object): return "VFS({})".format( ", ".join( "{}={!r}".format(k, self.__dict__[k]) - for k in "realpath vpath uread uwrite flags".split() + for k in "realpath vpath uread uwrite uadm flags".split() ) ) @@ -52,6 +53,7 @@ class VFS(object): "{}/{}".format(self.vpath, name).lstrip("/"), self.uread, self.uwrite, + self.uadm, self.flags, ) self._trk(vn) @@ -226,15 +228,19 @@ class VFS(object): for f in [{"vp": v, "ap": a, "st": n[1]} for v, a, n in files]: yield f - def user_tree(self, uname, readable=False, writable=False): + def user_tree(self, uname, readable=False, writable=False, admin=False): ret = [] opt1 = readable and (uname in self.uread or "*" in self.uread) opt2 = writable and (uname in self.uwrite or "*" in self.uwrite) - if opt1 or opt2: - ret.append(self.vpath) + if admin: + if opt1 and opt2: + ret.append(self.vpath) + else: + if opt1 or opt2: + ret.append(self.vpath) for _, vn in sorted(self.nodes.items()): - ret.extend(vn.user_tree(uname, readable, writable)) + ret.extend(vn.user_tree(uname, readable, writable, admin)) return ret @@ -269,7 +275,7 @@ class AuthSrv(object): yield prev, True - def _parse_config_file(self, fd, user, mread, mwrite, mflags, mount): + def _parse_config_file(self, fd, user, mread, mwrite, madm, mflags, mount): vol_src = None vol_dst = None self.line_ctr = 0 @@ -301,6 +307,7 @@ class AuthSrv(object): mount[vol_dst] = vol_src mread[vol_dst] = [] mwrite[vol_dst] = [] + madm[vol_dst] = [] mflags[vol_dst] = {} continue @@ -311,10 +318,15 @@ class AuthSrv(object): uname = "*" self._read_vol_str( - lvl, uname, mread[vol_dst], mwrite[vol_dst], mflags[vol_dst] + lvl, + uname, + mread[vol_dst], + mwrite[vol_dst], + madm[vol_dst], + mflags[vol_dst], ) - def _read_vol_str(self, lvl, uname, mr, mw, mf): + def _read_vol_str(self, lvl, uname, mr, mw, ma, mf): if lvl == "c": cval = True if "=" in uname: @@ -332,6 +344,9 @@ class AuthSrv(object): if lvl in "wa": mw.append(uname) + if lvl == "a": + ma.append(uname) + def _read_volflag(self, flags, name, value, is_list): if name not in ["mtp"]: flags[name] = value @@ -355,6 +370,7 @@ class AuthSrv(object): user = {} # username:password mread = {} # mountpoint:[username] mwrite = {} # mountpoint:[username] + madm = {} # mountpoint:[username] mflags = {} # mountpoint:[flag] mount = {} # dst:src (mountpoint:realpath) @@ -378,17 +394,22 @@ class AuthSrv(object): mount[dst] = src mread[dst] = [] mwrite[dst] = [] + madm[dst] = [] mflags[dst] = {} perms = perms.split(":") for (lvl, uname) in [[x[0], x[1:]] for x in perms]: - self._read_vol_str(lvl, uname, mread[dst], mwrite[dst], mflags[dst]) + self._read_vol_str( + lvl, uname, mread[dst], mwrite[dst], madm[dst], mflags[dst] + ) if self.args.c: for cfg_fn in self.args.c: with open(cfg_fn, "rb") as f: try: - self._parse_config_file(f, user, mread, mwrite, mflags, mount) + self._parse_config_file( + f, user, mread, mwrite, madm, mflags, mount + ) except: m = "\n\033[1;31m\nerror in config file {} on line {}:\n\033[0m" print(m.format(cfg_fn, self.line_ctr)) @@ -410,12 +431,15 @@ class AuthSrv(object): if dst == "": # rootfs was mapped; fully replaces the default CWD vfs - vfs = VFS(mount[dst], dst, mread[dst], mwrite[dst], mflags[dst]) + vfs = VFS( + mount[dst], dst, mread[dst], mwrite[dst], madm[dst], mflags[dst] + ) continue v = vfs.add(mount[dst], dst) v.uread = mread[dst] v.uwrite = mwrite[dst] + v.uadm = madm[dst] v.flags = mflags[dst] missing_users = {} diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index 4e525628..f2519c89 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -10,6 +10,7 @@ import json import string import socket import ctypes +import traceback from datetime import datetime import calendar @@ -155,6 +156,7 @@ class HttpCli(object): if self.uname: self.rvol = self.auth.vfs.user_tree(self.uname, readable=True) self.wvol = self.auth.vfs.user_tree(self.uname, writable=True) + self.avol = self.auth.vfs.user_tree(self.uname, True, True, True) ua = self.headers.get("user-agent", "") self.is_rclone = ua.startswith("rclone/") @@ -326,6 +328,12 @@ class HttpCli(object): self.vpath = None return self.tx_mounts() + if "scan" in self.uparam: + return self.scanvol() + + if "stack" in self.uparam: + return self.tx_stack() + return self.tx_browser() def handle_options(self): @@ -1304,10 +1312,61 @@ class HttpCli(object): suf = self.urlq(rm=["h"]) rvol = [x + "/" if x else x for x in self.rvol] wvol = [x + "/" if x else x for x in self.wvol] - html = self.j2("splash", this=self, rvol=rvol, wvol=wvol, url_suf=suf) + + vstate = {} + if self.avol and not self.args.no_rescan: + x = self.conn.hsrv.broker.put(True, "up2k.get_volstate") + vstate = json.loads(x.get()) + + html = self.j2( + "splash", + this=self, + rvol=rvol, + wvol=wvol, + avol=self.avol, + vstate=vstate, + url_suf=suf, + ) self.reply(html.encode("utf-8"), headers=NO_STORE) return True + def scanvol(self): + if not self.readable or not self.writable: + raise Pebkac(403, "not admin") + + if self.args.no_rescan: + raise Pebkac(403, "disabled by argv") + + vn, _ = self.auth.vfs.get(self.vpath, self.uname, True, True) + + args = [self.auth.vfs.all_vols, [vn.vpath]] + x = self.conn.hsrv.broker.put(True, "up2k.rescan", *args) + x = x.get() + if not x: + self.redirect("", "?h") + return "" + + raise Pebkac(500, x) + + def tx_stack(self): + if not self.readable or not self.writable: + raise Pebkac(403, "not admin") + + if self.args.no_stack: + raise Pebkac(403, "disabled by argv") + + ret = [] + names = dict([(t.ident, t.name) for t in threading.enumerate()]) + for tid, stack in sys._current_frames().items(): + ret.append("\n\n# {} ({:x})".format(names.get(tid), tid)) + for fn, lno, name, line in traceback.extract_stack(stack): + ret.append('File: "{}", line {}, in {}'.format(fn, lno, name)) + if line: + ret.append(" " + str(line.strip())) + + ret = ("
" + "\n".join(ret)).encode("utf-8")
+ self.reply(ret)
+
def tx_tree(self):
top = self.uparam["tree"] or ""
dst = self.vpath
diff --git a/copyparty/up2k.py b/copyparty/up2k.py
index 0617b5a5..3a4fe24a 100644
--- a/copyparty/up2k.py
+++ b/copyparty/up2k.py
@@ -52,7 +52,6 @@ class Up2k(object):
self.hub = hub
self.args = hub.args
self.log_func = hub.log
- self.all_vols = all_vols
# config
self.salt = self.args.salt
@@ -61,12 +60,14 @@ class Up2k(object):
self.mutex = threading.Lock()
self.hashq = Queue()
self.tagq = Queue()
+ self.volstate = {}
self.registry = {}
self.entags = {}
self.flags = {}
self.cur = {}
self.mtag = None
self.pending_tags = None
+ self.mtp_parsers = {}
self.mem_cur = None
self.sqlite_ver = None
@@ -92,7 +93,15 @@ class Up2k(object):
if not HAVE_SQLITE3:
self.log("could not initialize sqlite3, will use in-memory registry only")
- have_e2d = self.init_indexes()
+ if self.args.no_fastboot:
+ self.deferred_init(all_vols)
+ else:
+ t = threading.Thread(target=self.deferred_init, args=(all_vols,))
+ t.daemon = True
+ t.start()
+
+ def deferred_init(self, all_vols):
+ have_e2d = self.init_indexes(all_vols)
if have_e2d:
thr = threading.Thread(target=self._snapshot)
@@ -115,6 +124,19 @@ class Up2k(object):
def log(self, msg, c=0):
self.log_func("up2k", msg + "\033[K", c)
+ def get_volstate(self):
+ return json.dumps(self.volstate, indent=4)
+
+ def rescan(self, all_vols, scan_vols):
+ if hasattr(self, "pp"):
+ return "cannot initiate; scan is already in progress"
+
+ args = (all_vols, scan_vols)
+ t = threading.Thread(target=self.init_indexes, args=args)
+ t.daemon = True
+ t.start()
+ return None
+
def _vis_job_progress(self, job):
perc = 100 - (len(job["need"]) * 100.0 / len(job["hash"]))
path = os.path.join(job["ptop"], job["prel"], job["name"])
@@ -137,9 +159,9 @@ class Up2k(object):
return True, ret
- def init_indexes(self):
+ def init_indexes(self, all_vols, scan_vols=[]):
self.pp = ProgressPrinter()
- vols = self.all_vols.values()
+ vols = all_vols.values()
t0 = time.time()
have_e2d = False
@@ -159,24 +181,32 @@ class Up2k(object):
for vol in vols:
try:
os.listdir(vol.realpath)
- live_vols.append(vol)
+ if not self.register_vpath(vol.realpath, vol.flags):
+ raise Exception()
+
+ if vol.vpath in scan_vols or not scan_vols:
+ live_vols.append(vol)
+
+ if vol.vpath not in self.volstate:
+ self.volstate[vol.vpath] = "OFFLINE (not initialized)"
except:
- self.log("cannot access " + vol.realpath, c=1)
+ # self.log("db not enabled for {}".format(m, vol.realpath))
+ pass
vols = live_vols
+ need_vac = {}
need_mtag = False
for vol in vols:
if "e2t" in vol.flags:
need_mtag = True
- if need_mtag:
+ if need_mtag and not self.mtag:
self.mtag = MTag(self.log_func, self.args)
if not self.mtag.usable:
self.mtag = None
- # e2ds(a) volumes first,
- # also covers tags where e2ts is set
+ # e2ds(a) volumes first
for vol in vols:
en = {}
if "mte" in vol.flags:
@@ -188,26 +218,45 @@ class Up2k(object):
have_e2d = True
if "e2ds" in vol.flags:
- r = self._build_file_index(vol, vols)
- if not r:
- needed_mutagen = True
+ self.volstate[vol.vpath] = "busy (hashing files)"
+ _, vac = self._build_file_index(vol, list(all_vols.values()))
+ if vac:
+ need_vac[vol] = True
+
+ if "e2ts" not in vol.flags:
+ m = "online, idle"
+ else:
+ m = "online (tags pending)"
+
+ self.volstate[vol.vpath] = m
# open the rest + do any e2ts(a)
needed_mutagen = False
for vol in vols:
- r = self.register_vpath(vol.realpath, vol.flags)
- if not r or "e2ts" not in vol.flags:
+ if "e2ts" not in vol.flags:
continue
- cur, db_path, sz0 = r
- n_add, n_rm, success = self._build_tags_index(vol.realpath)
+ m = "online (reading tags)"
+ self.volstate[vol.vpath] = m
+ self.log("{} [{}]".format(m, vol.realpath))
+
+ nadd, nrm, success = self._build_tags_index(vol)
if not success:
needed_mutagen = True
- if n_add or n_rm:
- self.vac(cur, db_path, n_add, n_rm, sz0)
+ if nadd or nrm:
+ need_vac[vol] = True
+
+ self.volstate[vol.vpath] = "online (mtp soon)"
+
+ for vol in need_vac:
+ cur, _ = self.register_vpath(vol.realpath, vol.flags)
+ with self.mutex:
+ cur.connection.commit()
+ cur.execute("vacuum")
self.pp.end = True
+
msg = "{} volumes in {:.2f} sec"
self.log(msg.format(len(vols), time.time() - t0))
@@ -215,110 +264,104 @@ class Up2k(object):
msg = "could not read tags because no backends are available (mutagen or ffprobe)"
self.log(msg, c=1)
+ thr = None
+ if self.mtag:
+ m = "online (running mtp)"
+ if scan_vols:
+ thr = threading.Thread(target=self._run_all_mtp)
+ thr.daemon = True
+ else:
+ del self.pp
+ m = "online, idle"
+
+ for vol in vols:
+ self.volstate[vol.vpath] = m
+
+ if thr:
+ thr.start()
+
return have_e2d
def register_vpath(self, ptop, flags):
- with self.mutex:
- if ptop in self.registry:
- return None
+ db_path = os.path.join(ptop, ".hist", "up2k.db")
+ if ptop in self.registry:
+ return [self.cur[ptop], db_path]
- _, flags = self._expr_idx_filter(flags)
+ _, flags = self._expr_idx_filter(flags)
- ft = "\033[0;32m{}{:.0}"
- ff = "\033[0;35m{}{:.0}"
- fv = "\033[0;36m{}:\033[1;30m{}"
- a = [
- (ft if v is True else ff if v is False else fv).format(k, str(v))
- for k, v in flags.items()
- ]
- if a:
- self.log(" ".join(sorted(a)) + "\033[0m")
+ ft = "\033[0;32m{}{:.0}"
+ ff = "\033[0;35m{}{:.0}"
+ fv = "\033[0;36m{}:\033[1;30m{}"
+ a = [
+ (ft if v is True else ff if v is False else fv).format(k, str(v))
+ for k, v in flags.items()
+ ]
+ if a:
+ self.log(" ".join(sorted(a)) + "\033[0m")
- reg = {}
- path = os.path.join(ptop, ".hist", "up2k.snap")
- if "e2d" in flags and os.path.exists(path):
- with gzip.GzipFile(path, "rb") as f:
- j = f.read().decode("utf-8")
+ reg = {}
+ path = os.path.join(ptop, ".hist", "up2k.snap")
+ if "e2d" in flags and os.path.exists(path):
+ with gzip.GzipFile(path, "rb") as f:
+ j = f.read().decode("utf-8")
- reg2 = json.loads(j)
- for k, job in reg2.items():
- path = os.path.join(job["ptop"], job["prel"], job["name"])
- if os.path.exists(fsenc(path)):
- reg[k] = job
- job["poke"] = time.time()
- else:
- self.log("ign deleted file in snap: [{}]".format(path))
+ reg2 = json.loads(j)
+ for k, job in reg2.items():
+ path = os.path.join(job["ptop"], job["prel"], job["name"])
+ if os.path.exists(fsenc(path)):
+ reg[k] = job
+ job["poke"] = time.time()
+ else:
+ self.log("ign deleted file in snap: [{}]".format(path))
- m = "loaded snap {} |{}|".format(path, len(reg.keys()))
- m = [m] + self._vis_reg_progress(reg)
- self.log("\n".join(m))
-
- self.flags[ptop] = flags
- self.registry[ptop] = reg
- if not HAVE_SQLITE3 or "e2d" not in flags or "d2d" in flags:
- return None
-
- try:
- os.mkdir(os.path.join(ptop, ".hist"))
- except:
- pass
-
- db_path = os.path.join(ptop, ".hist", "up2k.db")
- if ptop in self.cur:
- return None
-
- try:
- sz0 = 0
- if os.path.exists(db_path):
- sz0 = os.path.getsize(db_path) // 1024
-
- cur = self._open_db(db_path)
- self.cur[ptop] = cur
- return [cur, db_path, sz0]
- except:
- msg = "cannot use database at [{}]:\n{}"
- self.log(msg.format(ptop, traceback.format_exc()))
+ m = "loaded snap {} |{}|".format(path, len(reg.keys()))
+ m = [m] + self._vis_reg_progress(reg)
+ self.log("\n".join(m))
+ self.flags[ptop] = flags
+ self.registry[ptop] = reg
+ if not HAVE_SQLITE3 or "e2d" not in flags or "d2d" in flags:
return None
+ try:
+ os.mkdir(os.path.join(ptop, ".hist"))
+ except:
+ pass
+
+ try:
+ cur = self._open_db(db_path)
+ self.cur[ptop] = cur
+ return [cur, db_path]
+ except:
+ msg = "cannot use database at [{}]:\n{}"
+ self.log(msg.format(ptop, traceback.format_exc()))
+
+ return None
+
def _build_file_index(self, vol, all_vols):
do_vac = False
top = vol.realpath
- reg = self.register_vpath(top, vol.flags)
- if not reg:
- return
+ with self.mutex:
+ cur, _ = self.register_vpath(top, vol.flags)
- _, db_path, sz0 = reg
- dbw = [reg[0], 0, time.time()]
- self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
+ dbw = [cur, 0, time.time()]
+ self.pp.n = next(dbw[0].execute("select count(w) from up"))[0]
- excl = [
- vol.realpath + "/" + d.vpath[len(vol.vpath) :].lstrip("/")
- for d in all_vols
- if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
- ]
- n_add = self._build_dir(dbw, top, set(excl), top)
- n_rm = self._drop_lost(dbw[0], top)
- if dbw[1]:
- self.log("commit {} new files".format(dbw[1]))
- dbw[0].connection.commit()
+ excl = [
+ vol.realpath + "/" + d.vpath[len(vol.vpath) :].lstrip("/")
+ for d in all_vols
+ if d != vol and (d.vpath.startswith(vol.vpath + "/") or not vol.vpath)
+ ]
+ if WINDOWS:
+ excl = [x.replace("/", "\\") for x in excl]
- n_add, n_rm, success = self._build_tags_index(vol.realpath)
+ n_add = self._build_dir(dbw, top, set(excl), top)
+ n_rm = self._drop_lost(dbw[0], top)
+ if dbw[1]:
+ self.log("commit {} new files".format(dbw[1]))
+ dbw[0].connection.commit()
- dbw[0].connection.commit()
- if n_add or n_rm or do_vac:
- self.vac(dbw[0], db_path, n_add, n_rm, sz0)
-
- return success
-
- def vac(self, cur, db_path, n_add, n_rm, sz0):
- sz1 = os.path.getsize(db_path) // 1024
- cur.execute("vacuum")
- sz2 = os.path.getsize(db_path) // 1024
- msg = "{} new, {} del, {} kB vacced, {} kB gain, {} kB now".format(
- n_add, n_rm, sz1 - sz2, sz2 - sz0, sz2
- )
- self.log(msg)
+ return True, n_add or n_rm or do_vac
def _build_dir(self, dbw, top, excl, cdir):
self.pp.msg = "a{} {}".format(self.pp.n, cdir)
@@ -413,45 +456,53 @@ class Up2k(object):
return len(rm)
- def _build_tags_index(self, ptop):
- entags = self.entags[ptop]
- flags = self.flags[ptop]
- cur = self.cur[ptop]
+ def _build_tags_index(self, vol):
+ ptop = vol.realpath
+ with self.mutex:
+ _, db_path = self.register_vpath(ptop, vol.flags)
+ entags = self.entags[ptop]
+ flags = self.flags[ptop]
+ cur = self.cur[ptop]
+
n_add = 0
n_rm = 0
n_buf = 0
last_write = time.time()
if "e2tsr" in flags:
- n_rm = cur.execute("select count(w) from mt").fetchone()[0]
- if n_rm:
- self.log("discarding {} media tags for a full rescan".format(n_rm))
- cur.execute("delete from mt")
- else:
- self.log("volume has e2tsr but there are no media tags to discard")
+ with self.mutex:
+ n_rm = cur.execute("select count(w) from mt").fetchone()[0]
+ if n_rm:
+ self.log("discarding {} media tags for a full rescan".format(n_rm))
+ cur.execute("delete from mt")
# integrity: drop tags for tracks that were deleted
if "e2t" in flags:
- drops = []
- c2 = cur.connection.cursor()
- up_q = "select w from up where substr(w,1,16) = ?"
- for (w,) in cur.execute("select w from mt"):
- if not c2.execute(up_q, (w,)).fetchone():
- drops.append(w[:16])
- c2.close()
+ with self.mutex:
+ drops = []
+ c2 = cur.connection.cursor()
+ up_q = "select w from up where substr(w,1,16) = ?"
+ for (w,) in cur.execute("select w from mt"):
+ if not c2.execute(up_q, (w,)).fetchone():
+ drops.append(w[:16])
+ c2.close()
- if drops:
- msg = "discarding media tags for {} deleted files"
- self.log(msg.format(len(drops)))
- n_rm += len(drops)
- for w in drops:
- cur.execute("delete from mt where w = ?", (w,))
+ if drops:
+ msg = "discarding media tags for {} deleted files"
+ self.log(msg.format(len(drops)))
+ n_rm += len(drops)
+ for w in drops:
+ cur.execute("delete from mt where w = ?", (w,))
# bail if a volume flag disables indexing
if "d2t" in flags or "d2d" in flags:
return n_add, n_rm, True
# add tags for new files
+ gcur = cur
+ with self.mutex:
+ gcur.connection.commit()
+
if "e2ts" in flags:
if not self.mtag:
return n_add, n_rm, False
@@ -460,8 +511,10 @@ class Up2k(object):
if self.mtag.prefer_mt and not self.args.no_mtag_mt:
mpool = self._start_mpool()
- c2 = cur.connection.cursor()
- c3 = cur.connection.cursor()
+ conn = sqlite3.connect(db_path, timeout=15)
+ cur = conn.cursor()
+ c2 = conn.cursor()
+ c3 = conn.cursor()
n_left = cur.execute("select count(w) from up").fetchone()[0]
for w, rd, fn in cur.execute("select w, rd, fn from up"):
n_left -= 1
@@ -483,7 +536,8 @@ class Up2k(object):
n_tags = self._tag_file(c3, *args)
else:
mpool.put(["mtag"] + args)
- n_tags = len(self._flush_mpool(c3))
+ with self.mutex:
+ n_tags = len(self._flush_mpool(c3))
n_add += n_tags
n_buf += n_tags
@@ -495,26 +549,32 @@ class Up2k(object):
last_write = time.time()
n_buf = 0
- self._stop_mpool(mpool, c3)
+ self._stop_mpool(mpool)
+ with self.mutex:
+ n_add += len(self._flush_mpool(c3))
+ conn.commit()
c3.close()
c2.close()
+ cur.close()
+ conn.close()
+
+ with self.mutex:
+ gcur.connection.commit()
return n_add, n_rm, True
def _flush_mpool(self, wcur):
- with self.mutex:
- ret = []
- for x in self.pending_tags:
- self._tag_file(wcur, *x)
- ret.append(x[1])
+ ret = []
+ for x in self.pending_tags:
+ self._tag_file(wcur, *x)
+ ret.append(x[1])
- self.pending_tags = []
- return ret
+ self.pending_tags = []
+ return ret
def _run_all_mtp(self):
t0 = time.time()
- self.mtp_parsers = {}
for ptop, flags in self.flags.items():
if "mtp" in flags:
self._run_one_mtp(ptop)
@@ -523,10 +583,11 @@ class Up2k(object):
msg = "mtp finished in {:.2f} sec ({})"
self.log(msg.format(td, s2hms(td, True)))
- def _run_one_mtp(self, ptop):
- db_path = os.path.join(ptop, ".hist", "up2k.db")
- sz0 = os.path.getsize(db_path) // 1024
+ del self.pp
+ for k in list(self.volstate.keys()):
+ self.volstate[k] = "online, idle"
+ def _run_one_mtp(self, ptop):
entags = self.entags[ptop]
parsers = {}
@@ -585,9 +646,8 @@ class Up2k(object):
jobs.append([parsers, None, w, abspath])
in_progress[w] = True
- done = self._flush_mpool(wcur)
-
with self.mutex:
+ done = self._flush_mpool(wcur)
for w in done:
to_delete[w] = True
in_progress.pop(w)
@@ -628,15 +688,16 @@ class Up2k(object):
with self.mutex:
cur.connection.commit()
- done = self._stop_mpool(mpool, wcur)
+ self._stop_mpool(mpool)
with self.mutex:
+ done = self._flush_mpool(wcur)
for w in done:
q = "delete from mt where w = ? and k = 't:mtp'"
cur.execute(q, (w,))
cur.connection.commit()
if n_done:
- self.vac(cur, db_path, n_done, 0, sz0)
+ cur.execute("vacuum")
wcur.close()
cur.close()
@@ -693,7 +754,7 @@ class Up2k(object):
return mpool
- def _stop_mpool(self, mpool, wcur):
+ def _stop_mpool(self, mpool):
if not mpool:
return
@@ -701,8 +762,6 @@ class Up2k(object):
mpool.put(None)
mpool.join()
- done = self._flush_mpool(wcur)
- return done
def _tag_thr(self, q):
while True:
diff --git a/copyparty/web/splash.css b/copyparty/web/splash.css
index 7d29f8cd..47a4a7f7 100644
--- a/copyparty/web/splash.css
+++ b/copyparty/web/splash.css
@@ -26,6 +26,13 @@ a {
border-radius: .2em;
padding: .2em .8em;
}
+td, th {
+ padding: .3em .6em;
+ text-align: left;
+}
+.btns {
+ margin: 1em 0;
+}
html.dark,
diff --git a/copyparty/web/splash.html b/copyparty/web/splash.html
index 9f95f035..acbffe62 100644
--- a/copyparty/web/splash.html
+++ b/copyparty/web/splash.html
@@ -13,6 +13,23 @@
hello {{ this.uname }}
+ {%- if avol %}
+ admin panel:
+
+ vol action status
+
+ {% for mp in avol %}
+ {%- if mp in vstate and vstate[mp] %}
+ /{{ mp }} rescan {{ vstate[mp] }}
+ {%- endif %}
+ {% endfor %}
+
+
+
+ dump stack
+
+ {%- endif %}
+
{%- if rvol %}
you can browse these: