From d44ea245307c06eb060b6e6f5c2cba91a759c940 Mon Sep 17 00:00:00 2001 From: ed Date: Tue, 10 Feb 2026 21:25:15 +0000 Subject: [PATCH] add fika --- copyparty/__main__.py | 1 + copyparty/authsrv.py | 16 +++++++++------ copyparty/svchub.py | 4 ++-- copyparty/up2k.py | 46 +++++++++++++++++++++++++++++++++++++++---- tests/util.py | 2 +- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/copyparty/__main__.py b/copyparty/__main__.py index d5ce2020..136a55d9 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -1777,6 +1777,7 @@ def add_db_general(ap, hcores): ap2.add_argument("-e2vp", action="store_true", help="on hash mismatch: panic and quit copyparty") ap2.add_argument("--hist", metavar="PATH", type=u, default="", help="where to store volume data (db, thumbs); default is a folder named \".hist\" inside each volume (volflag=hist)") ap2.add_argument("--dbpath", metavar="PATH", type=u, default="", help="override where the volume databases are to be placed; default is the same as \033[33m--hist\033[0m (volflag=dbpath)") + ap2.add_argument("--fika", metavar="TXT", type=u, default="ucd", help="list of user-actions to allow while filesystem-indexer is still busy; set blank to never interrupt indexing (old default). NOTE: blank is recommended if dedup enabled. [\033[32mu\033[0m]=uploads, [\033[32mc\033[0m]=filecopy, [\033[32mm\033[0m]=move/rename, [\033[32md\033[0m]=delete") ap2.add_argument("--no-hash", metavar="PTN", type=u, default="", help="regex: disable hashing of matching absolute-filesystem-paths during e2ds folder scans (must be specified as one big regex, not multiple times) (volflag=nohash)") ap2.add_argument("--no-idx", metavar="PTN", type=u, default=noidx, help="regex: disable indexing of matching absolute-filesystem-paths during e2ds folder scan (must be specified as one big regex, not multiple times) (volflag=noidx)") ap2.add_argument("--no-dirsz", action="store_true", help="do not show total recursive size of folders in listings, show inode size instead; slightly faster (volflag=nodirsz)") diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 2c97a8fc..1921bf40 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -2893,6 +2893,7 @@ class AuthSrv(object): have_e2d = False have_e2t = False have_dedup = False + have_symdup = False unsafe_dedup = [] t = "volumes and permissions:\n" for zv in vfs.all_vols.values(): @@ -2932,15 +2933,18 @@ class AuthSrv(object): if "dedup" in zv.flags: have_dedup = True - if ( - "e2d" not in zv.flags - and "hardlink" not in zv.flags - and "reflink" not in zv.flags - ): - unsafe_dedup.append("/" + zv.vpath) + if "hardlink" not in zv.flags and "reflink" not in zv.flags: + have_symdup = True + if "e2d" not in zv.flags: + unsafe_dedup.append("/" + zv.vpath) t += "\n" + if have_symdup and self.args.fika: + t = "WARNING: disabling fika due to symlink-based dedup in at least one volume; consider --reflink or --hardlink" + # self.args.fika = self.args.fika.replace("m", "").replace("d", "") # probably not enough + self.args.fika = "" + if self.warn_anonwrite and verbosity > 4: if not self.args.no_voldump: self.log(t) diff --git a/copyparty/svchub.py b/copyparty/svchub.py index 072f9928..cf0340e9 100644 --- a/copyparty/svchub.py +++ b/copyparty/svchub.py @@ -1160,9 +1160,9 @@ class SvcHub(object): elif al.ban_url == "no": al.sus_urls = None - zs = "idp_h_grp idp_h_key pw_hdr pw_urlp xf_host xf_proto xf_proto_fb xff_hdr" + zs = "fika idp_h_grp idp_h_key pw_hdr pw_urlp xf_host xf_proto xf_proto_fb xff_hdr" for k in zs.split(" "): - setattr(al, k, getattr(al, k).lower()) + setattr(al, k, str(getattr(al, k)).lower().strip()) al.idp_h_usr = [x.lower() for x in al.idp_h_usr or []] diff --git a/copyparty/up2k.py b/copyparty/up2k.py index a3da35ab..7d525fa5 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -166,6 +166,7 @@ class Up2k(object): self.gt0 = 0 self.gt1 = 0 self.stop = False + self.fika = "" self.mutex = threading.Lock() self.reload_mutex = threading.Lock() self.reload_flag = 0 @@ -1475,6 +1476,23 @@ class Up2k(object): return True, bool(n_add or n_rm or do_vac) + def _fika(self, db: Dbw) -> None: + zs = self.fika + self.fika = "" + if zs not in self.args.fika: + return + + t = "fika(%s); commit %d new files; %d updates" + self.log(t % (zs, db.nf, db.n)) + db.c.connection.commit() + db.n = db.nf = 0 + db.t = time.time() + + self.mutex.release() + time.sleep(0.5) + self.mutex.acquire() + db.t = time.time() + def _build_dir( self, db: Dbw, @@ -1527,8 +1545,12 @@ class Up2k(object): gl = sorted(g) partials = set([x[0] for x in gl if "PARTIAL" in x[0]]) for iname, inf in gl: - if self.stop: - return -1, 0, 0 + if self.fika: + if not self.stop: + self._fika(db) + if self.stop: + self.fika = "f" + return -1, 0, 0 rp = rds + iname abspath = cdirs + iname @@ -1675,8 +1697,12 @@ class Up2k(object): seen_files = set([x[2] for x in files]) # for dropcheck for sz, lmod, fn in files: - if self.stop: - return -1, 0, 0 + if self.fika: + if not self.stop: + self._fika(db) + if self.stop: + self.fika = "f" + return -1, 0, 0 rp = rds + fn abspath = cdirs + fn @@ -2978,6 +3004,7 @@ class Up2k(object): raise Pebkac(503, SBUSY % ("fs-reload",)) got_lock = False + self.fika = "u" try: # bit expensive; 3.9=10x 3.11=2x if self.mutex.acquire(timeout=10): @@ -3643,6 +3670,7 @@ class Up2k(object): def handle_chunks( self, ptop: str, wark: str, chashes: list[str] ) -> tuple[list[str], int, list[list[int]], str, float, int, bool]: + self.fika = "u" with self.mutex, self.reg_mutex: self.db_act = self.vol_act[ptop] = time.time() job = self.registry[ptop].get(wark) @@ -3745,6 +3773,7 @@ class Up2k(object): def confirm_chunks( self, ptop: str, wark: str, written: list[str], locked: list[str] ) -> tuple[int, str]: + self.fika = "u" with self.mutex, self.reg_mutex: return self._confirm_chunks(ptop, wark, written, locked, True) @@ -3784,6 +3813,7 @@ class Up2k(object): def finish_upload(self, ptop: str, wark: str, busy_aps: dict[str, int]) -> None: self.busy_aps = busy_aps + self.fika = "u" with self.mutex, self.reg_mutex: self._finish_upload(ptop, wark) @@ -4138,6 +4168,7 @@ class Up2k(object): vn0, rem0 = self.vfs.get(vpath, uname, *permsets[0]) vn, rem = vn0.get_dbv(rem0) ptop = vn.realpath + self.fika = "d" with self.mutex, self.reg_mutex: abrt_cfg = vn.flags.get("u2abort", 1) addr = (ip or "\n") if abrt_cfg in (1, 2) else "" @@ -4266,6 +4297,7 @@ class Up2k(object): continue n_files += 1 + self.fika = "d" with self.mutex, self.reg_mutex: cur = None try: @@ -4330,6 +4362,7 @@ class Up2k(object): raise Pebkac(400, "file does not exist case-sensitively") if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): + self.fika = "c" with self.mutex: try: ret = self._cp_file(uname, ip, svp, dvp, curs) @@ -4351,6 +4384,7 @@ class Up2k(object): # don't use svn_dbv; would skip subvols due to _ls `if not rem:` g = svn.walk("", srem, [], uname, permsets, dots, scandir, True) + self.fika = "c" with self.mutex: try: for dbv, vrem, _, atop, files, rd, vd in g: @@ -4556,6 +4590,7 @@ class Up2k(object): raise Pebkac(400, "file does not exist case-sensitively") if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): + self.fika = "m" with self.mutex: try: ret = self._mv_file(uname, ip, svp, dvp, curs) @@ -4579,6 +4614,7 @@ class Up2k(object): raise Pebkac(400, "mv: source folder contains other volumes") g = svn.walk("", srem, [], uname, permsets, 2, scandir, True) + self.fika = "m" with self.mutex: try: for dbv, vrem, _, atop, files, rd, vd in g: @@ -5317,6 +5353,7 @@ class Up2k(object): self.do_snapshot() def do_snapshot(self) -> None: + self.fika = "u" with self.mutex, self.reg_mutex: for k, reg in self.registry.items(): self._snap_reg(k, reg) @@ -5597,6 +5634,7 @@ class Up2k(object): def shutdown(self) -> None: self.stop = True + self.fika = "f" if self.mth: self.mth.stop = True diff --git a/tests/util.py b/tests/util.py index 7c0b6de1..d2a8f476 100644 --- a/tests/util.py +++ b/tests/util.py @@ -164,7 +164,7 @@ class Cfg(Namespace): ex = "ctl_re db_act forget_ip idp_cookie idp_store k304 loris no304 nosubtle qr_pin qr_wait re_maxage rproxy rsp_jtr rsp_slp s_wr_slp snap_wri theme themes turbo u2ow zipmaxn zipmaxs" ka.update(**{k: 0 for k in ex.split()}) - ex = "ah_alg bname chdir chmod_f chpw_db db_xattr doctitle df epilogues exit favico ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles txt_eol ufavico ufavico_h unlist up_site vname xff_src zipmaxt R RS SR" + ex = "ah_alg bname chdir chmod_f chpw_db db_xattr doctitle df epilogues exit favico fika ipa ipar html_head html_head_d html_head_s idp_login idp_logout lg_sba lg_sbf log_date log_fk md_sba md_sbf name og_desc og_site og_th og_title og_title_a og_title_v og_title_i opds_exts preadmes prologues readmes shr shr1 shr_site site smsg tcolor textfiles txt_eol ufavico ufavico_h unlist up_site vname xff_src zipmaxt R RS SR" ka.update(**{k: "" for k in ex.split()}) ex = "apnd_who ban_403 ban_404 ban_422 ban_pw ban_pwc ban_url dont_ban cachectl http_vary rcm rss_fmt_d rss_fmt_t spinner"