Atualizar src-py/renatoxsr/scripts/bulk_jsonl.py

This commit is contained in:
2026-09-26 15:17:14 -03:00
parent 9c9bc243b3
commit 77bb8efdfa
+11 -7
View File
@@ -14,22 +14,26 @@ if "--debug" in sys.argv:
def parse_args():
p = argparse.ArgumentParser()
p.add_argument("--src", default=os.getcwd())
p.add_argument("--outdir", default=os.getcwd())
p.add_argument("--outfile",
default=dt.datetime.now().isoformat().replace(":", "_")+".jsonl",
help="output file (default: `datetime.now().isoformat()`.jsonl)")
p.add_argument("--input", default=os.getcwd(), help="input folder")
p.add_argument("--glob", default="*")
p.add_argument("--overwrite", default=False)
p.add_argument("--logfile", default=sys.stderr, help="Log output(default: sys.stderr)")
p.add_argument("--loglevel", default="INFO", help="Log level (default: INFO)")
p.add_argument("filename",
default=dt.datetime.now().isoformat().replace(":", "_"),
help="output file (default: `datetime.now().isoformat()`)")
p.add_argument("filename", nargs="*", help="input file or glob pattern (default: `datetime.now().isoformat()`)")
return p.parse_args()
def main():
parsed_args = parse_args()
logger.setLevel(parsed_args.loglevel)
with open(Path(parsed_args.filename), "w" if parsed_args.overwrite else "a") as output_file:
for fpath in Path(parsed_args.src).glob(parsed_args.glob):
process_files = list()#Path(parsed_args.input).glob(parsed_args.glob))
process_files.extend(parsed_args.filename)
with open(Path(parsed_args.outdir) / parsed_args.outfile, "w" if parsed_args.overwrite else "a") as output_file:
for file in process_files:
fpath = Path(file)
data = {"hash": hash_b58(fpath), "fpath": str(fpath), "markdown": pandoc(fpath)}
if not data["markdown"]:
logger.error("No markdown output from '%s' (hash:%s)", fpath, data["hash"])