Atualizar src-py/renatoxsr/scripts/bulk_jsonl.py
This commit is contained in:
@@ -14,22 +14,26 @@ if "--debug" in sys.argv:
|
|||||||
|
|
||||||
def parse_args():
|
def parse_args():
|
||||||
p = argparse.ArgumentParser()
|
p = argparse.ArgumentParser()
|
||||||
p.add_argument("--src", default=os.getcwd())
|
p.add_argument("--outdir", default=os.getcwd())
|
||||||
|
p.add_argument("--outfile",
|
||||||
|
default=dt.datetime.now().isoformat().replace(":", "_")+".jsonl",
|
||||||
|
help="output file (default: `datetime.now().isoformat()`.jsonl)")
|
||||||
|
p.add_argument("--input", default=os.getcwd(), help="input folder")
|
||||||
p.add_argument("--glob", default="*")
|
p.add_argument("--glob", default="*")
|
||||||
p.add_argument("--overwrite", default=False)
|
p.add_argument("--overwrite", default=False)
|
||||||
p.add_argument("--logfile", default=sys.stderr, help="Log output(default: sys.stderr)")
|
p.add_argument("--logfile", default=sys.stderr, help="Log output(default: sys.stderr)")
|
||||||
p.add_argument("--loglevel", default="INFO", help="Log level (default: INFO)")
|
p.add_argument("--loglevel", default="INFO", help="Log level (default: INFO)")
|
||||||
p.add_argument("filename",
|
p.add_argument("filename", nargs="*", help="input file or glob pattern (default: `datetime.now().isoformat()`)")
|
||||||
default=dt.datetime.now().isoformat().replace(":", "_"),
|
|
||||||
help="output file (default: `datetime.now().isoformat()`)")
|
|
||||||
return p.parse_args()
|
return p.parse_args()
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parsed_args = parse_args()
|
parsed_args = parse_args()
|
||||||
logger.setLevel(parsed_args.loglevel)
|
logger.setLevel(parsed_args.loglevel)
|
||||||
|
process_files = list()#Path(parsed_args.input).glob(parsed_args.glob))
|
||||||
with open(Path(parsed_args.filename), "w" if parsed_args.overwrite else "a") as output_file:
|
process_files.extend(parsed_args.filename)
|
||||||
for fpath in Path(parsed_args.src).glob(parsed_args.glob):
|
with open(Path(parsed_args.outdir) / parsed_args.outfile, "w" if parsed_args.overwrite else "a") as output_file:
|
||||||
|
for file in process_files:
|
||||||
|
fpath = Path(file)
|
||||||
data = {"hash": hash_b58(fpath), "fpath": str(fpath), "markdown": pandoc(fpath)}
|
data = {"hash": hash_b58(fpath), "fpath": str(fpath), "markdown": pandoc(fpath)}
|
||||||
if not data["markdown"]:
|
if not data["markdown"]:
|
||||||
logger.error("No markdown output from '%s' (hash:%s)", fpath, data["hash"])
|
logger.error("No markdown output from '%s' (hash:%s)", fpath, data["hash"])
|
||||||
|
|||||||
Reference in New Issue
Block a user