Atualizar src-py/renatoxsr/scripts/bulk_jsonl.py
This commit is contained in:
@@ -14,22 +14,26 @@ if "--debug" in sys.argv:
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--src", default=os.getcwd())
|
||||
p.add_argument("--outdir", default=os.getcwd())
|
||||
p.add_argument("--outfile",
|
||||
default=dt.datetime.now().isoformat().replace(":", "_")+".jsonl",
|
||||
help="output file (default: `datetime.now().isoformat()`.jsonl)")
|
||||
p.add_argument("--input", default=os.getcwd(), help="input folder")
|
||||
p.add_argument("--glob", default="*")
|
||||
p.add_argument("--overwrite", default=False)
|
||||
p.add_argument("--logfile", default=sys.stderr, help="Log output(default: sys.stderr)")
|
||||
p.add_argument("--loglevel", default="INFO", help="Log level (default: INFO)")
|
||||
p.add_argument("filename",
|
||||
default=dt.datetime.now().isoformat().replace(":", "_"),
|
||||
help="output file (default: `datetime.now().isoformat()`)")
|
||||
p.add_argument("filename", nargs="*", help="input file or glob pattern (default: `datetime.now().isoformat()`)")
|
||||
return p.parse_args()
|
||||
|
||||
def main():
|
||||
parsed_args = parse_args()
|
||||
logger.setLevel(parsed_args.loglevel)
|
||||
|
||||
with open(Path(parsed_args.filename), "w" if parsed_args.overwrite else "a") as output_file:
|
||||
for fpath in Path(parsed_args.src).glob(parsed_args.glob):
|
||||
process_files = list()#Path(parsed_args.input).glob(parsed_args.glob))
|
||||
process_files.extend(parsed_args.filename)
|
||||
with open(Path(parsed_args.outdir) / parsed_args.outfile, "w" if parsed_args.overwrite else "a") as output_file:
|
||||
for file in process_files:
|
||||
fpath = Path(file)
|
||||
data = {"hash": hash_b58(fpath), "fpath": str(fpath), "markdown": pandoc(fpath)}
|
||||
if not data["markdown"]:
|
||||
logger.error("No markdown output from '%s' (hash:%s)", fpath, data["hash"])
|
||||
|
||||
Reference in New Issue
Block a user