add fika

copyparty/__main__.py

@@ -1777,6 +1777,7 @@ def add_db_general(ap, hcores):
     ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty")
     ap2.add_argument("--hist", metavar="PATH", type=u, default="", help="where to store volume data (db, thumbs); default is a folder named \".hist\" inside each volume (volflag=hist)")
     ap2.add_argument("--dbpath", metavar="PATH", type=u, default="", help="override where the volume databases are to be placed; default is the same as \033[33m--hist\033[0m (volflag=dbpath)")
+    ap2.add_argument("--fika", metavar="TXT", type=u, default="ucd", help="list of user-actions to allow while filesystem-indexer is still busy; set blank to never interrupt indexing (old default). NOTE: blank is recommended if dedup enabled. [\033[32mu\033[0m]=uploads, [\033[32mc\033[0m]=filecopy, [\033[32mm\033[0m]=move/rename, [\033[32md\033[0m]=delete")
     ap2.add_argument("--no-hash", metavar="PTN", type=u, default="", help="regex: disable hashing of matching absolute-filesystem-paths during e2ds folder scans (must be specified as one big regex, not multiple times) (volflag=nohash)")
     ap2.add_argument("--no-idx", metavar="PTN", type=u, default=noidx, help="regex: disable indexing of matching absolute-filesystem-paths during e2ds folder scan (must be specified as one big regex, not multiple times) (volflag=noidx)")
     ap2.add_argument("--no-dirsz", action="store_true", help="do not show total recursive size of folders in listings, show inode size instead; slightly faster (volflag=nodirsz)")

copyparty/authsrv.py

@@ -2893,6 +2893,7 @@ class AuthSrv(object):
         have_e2d = False
         have_e2t = False
         have_dedup = False
+        have_symdup = False
         unsafe_dedup = []
         t = "volumes and permissions:\n"
         for zv in vfs.all_vols.values():
@@ -2932,15 +2933,18 @@ class AuthSrv(object):
 
             if "dedup" in zv.flags:
                 have_dedup = True
-                if (
-                    "e2d" not in zv.flags
-                    and "hardlink" not in zv.flags
-                    and "reflink" not in zv.flags
-                ):
-                    unsafe_dedup.append("/" + zv.vpath)
+                if "hardlink" not in zv.flags and "reflink" not in zv.flags:
+                    have_symdup = True
+                    if "e2d" not in zv.flags:
+                        unsafe_dedup.append("/" + zv.vpath)
 
             t += "\n"
 
+        if have_symdup and self.args.fika:
+            t = "WARNING: disabling fika due to symlink-based dedup in at least one volume; consider --reflink or --hardlink"
+            # self.args.fika = self.args.fika.replace("m", "").replace("d", "")  # probably not enough
+            self.args.fika = ""
+
         if self.warn_anonwrite and verbosity > 4:
             if not self.args.no_voldump:
                 self.log(t)

copyparty/svchub.py

@@ -1160,9 +1160,9 @@ class SvcHub(object):
         elif al.ban_url == "no":
             al.sus_urls = None
 
-        zs = "idp_h_grp idp_h_key pw_hdr pw_urlp xf_host xf_proto xf_proto_fb xff_hdr"
+        zs = "fika idp_h_grp idp_h_key pw_hdr pw_urlp xf_host xf_proto xf_proto_fb xff_hdr"
         for k in zs.split(" "):
-            setattr(al, k, getattr(al, k).lower())
+            setattr(al, k, str(getattr(al, k)).lower().strip())
 
         al.idp_h_usr = [x.lower() for x in al.idp_h_usr or []]
 

copyparty/up2k.py

@@ -166,6 +166,7 @@ class Up2k(object):
         self.gt0 = 0
         self.gt1 = 0
         self.stop = False
+        self.fika = ""
         self.mutex = threading.Lock()
         self.reload_mutex = threading.Lock()
         self.reload_flag = 0
@@ -1475,6 +1476,23 @@ class Up2k(object):
 
             return True, bool(n_add or n_rm or do_vac)
 
+    def _fika(self, db: Dbw) -> None:
+        zs = self.fika
+        self.fika = ""
+        if zs not in self.args.fika:
+            return
+
+        t = "fika(%s); commit %d new files; %d updates"
+        self.log(t % (zs, db.nf, db.n))
+        db.c.connection.commit()
+        db.n = db.nf = 0
+        db.t = time.time()
+
+        self.mutex.release()
+        time.sleep(0.5)
+        self.mutex.acquire()
+        db.t = time.time()
+
     def _build_dir(
         self,
         db: Dbw,
@@ -1527,8 +1545,12 @@ class Up2k(object):
         gl = sorted(g)
         partials = set([x[0] for x in gl if "PARTIAL" in x[0]])
         for iname, inf in gl:
-            if self.stop:
-                return -1, 0, 0
+            if self.fika:
+                if not self.stop:
+                    self._fika(db)
+                if self.stop:
+                    self.fika = "f"
+                    return -1, 0, 0
 
             rp = rds + iname
             abspath = cdirs + iname
@@ -1675,8 +1697,12 @@ class Up2k(object):
 
         seen_files = set([x[2] for x in files])  # for dropcheck
         for sz, lmod, fn in files:
-            if self.stop:
-                return -1, 0, 0
+            if self.fika:
+                if not self.stop:
+                    self._fika(db)
+                if self.stop:
+                    self.fika = "f"
+                    return -1, 0, 0
 
             rp = rds + fn
             abspath = cdirs + fn
@@ -2978,6 +3004,7 @@ class Up2k(object):
             raise Pebkac(503, SBUSY % ("fs-reload",))
 
         got_lock = False
+        self.fika = "u"
         try:
             # bit expensive; 3.9=10x 3.11=2x
             if self.mutex.acquire(timeout=10):
@@ -3643,6 +3670,7 @@ class Up2k(object):
     def handle_chunks(
         self, ptop: str, wark: str, chashes: list[str]
     ) -> tuple[list[str], int, list[list[int]], str, float, int, bool]:
+        self.fika = "u"
         with self.mutex, self.reg_mutex:
             self.db_act = self.vol_act[ptop] = time.time()
             job = self.registry[ptop].get(wark)
@@ -3745,6 +3773,7 @@ class Up2k(object):
     def confirm_chunks(
         self, ptop: str, wark: str, written: list[str], locked: list[str]
     ) -> tuple[int, str]:
+        self.fika = "u"
         with self.mutex, self.reg_mutex:
             return self._confirm_chunks(ptop, wark, written, locked, True)
 
@@ -3784,6 +3813,7 @@ class Up2k(object):
 
     def finish_upload(self, ptop: str, wark: str, busy_aps: dict[str, int]) -> None:
         self.busy_aps = busy_aps
+        self.fika = "u"
         with self.mutex, self.reg_mutex:
             self._finish_upload(ptop, wark)
 
@@ -4138,6 +4168,7 @@ class Up2k(object):
             vn0, rem0 = self.vfs.get(vpath, uname, *permsets[0])
             vn, rem = vn0.get_dbv(rem0)
             ptop = vn.realpath
+            self.fika = "d"
             with self.mutex, self.reg_mutex:
                 abrt_cfg = vn.flags.get("u2abort", 1)
                 addr = (ip or "\n") if abrt_cfg in (1, 2) else ""
@@ -4266,6 +4297,7 @@ class Up2k(object):
                         continue
 
                 n_files += 1
+                self.fika = "d"
                 with self.mutex, self.reg_mutex:
                     cur = None
                     try:
@@ -4330,6 +4362,7 @@ class Up2k(object):
             raise Pebkac(400, "file does not exist case-sensitively")
 
         if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
+            self.fika = "c"
             with self.mutex:
                 try:
                     ret = self._cp_file(uname, ip, svp, dvp, curs)
@@ -4351,6 +4384,7 @@ class Up2k(object):
 
         # don't use svn_dbv; would skip subvols due to _ls `if not rem:`
         g = svn.walk("", srem, [], uname, permsets, dots, scandir, True)
+        self.fika = "c"
         with self.mutex:
             try:
                 for dbv, vrem, _, atop, files, rd, vd in g:
@@ -4556,6 +4590,7 @@ class Up2k(object):
             raise Pebkac(400, "file does not exist case-sensitively")
 
         if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
+            self.fika = "m"
             with self.mutex:
                 try:
                     ret = self._mv_file(uname, ip, svp, dvp, curs)
@@ -4579,6 +4614,7 @@ class Up2k(object):
                 raise Pebkac(400, "mv: source folder contains other volumes")
 
         g = svn.walk("", srem, [], uname, permsets, 2, scandir, True)
+        self.fika = "m"
         with self.mutex:
             try:
                 for dbv, vrem, _, atop, files, rd, vd in g:
@@ -5317,6 +5353,7 @@ class Up2k(object):
                 self.do_snapshot()
 
     def do_snapshot(self) -> None:
+        self.fika = "u"
         with self.mutex, self.reg_mutex:
             for k, reg in self.registry.items():
                 self._snap_reg(k, reg)
@@ -5597,6 +5634,7 @@ class Up2k(object):
 
     def shutdown(self) -> None:
         self.stop = True
+        self.fika = "f"
 
         if self.mth:
             self.mth.stop = True

tests/util.py

@@ -164,7 +164,7 @@ class Cfg(Namespace):
         ex = "ctl_re db_act forget_ip idp_cookie idp_store k304 loris no304 nosubtle qr_pin qr_wait re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs"
         ka.update(**{k: 0 for k in ex.split()})
 
-        ex = "ah_alg bname chdir chmod_f chpw_db db_xattr doctitle df epilogues exit favico ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles txt_eol ufavico ufavico_h unlist up_site vname xff_src zipmaxt R RS SR"
+        ex = "ah_alg bname chdir chmod_f chpw_db db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles txt_eol ufavico ufavico_h unlist up_site vname xff_src zipmaxt R RS SR"
         ka.update(**{k: "" for k in ex.split()})
 
         ex = "apnd_who ban_403 ban_404 ban_422 ban_pw ban_pwc ban_url dont_ban cachectl http_vary rcm rss_fmt_d rss_fmt_t spinner"