From 8dc600edc87afdad829f0654d3c192d050baa658 Mon Sep 17 00:00:00 2001 From: Renato Rosa Date: Sat, 26 Sep 2026 14:24:29 -0300 Subject: [PATCH] Adicionar src-py/renatoxsr/scripts/bulk_jsonl.py --- src-py/renatoxsr/scripts/bulk_jsonl.py | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src-py/renatoxsr/scripts/bulk_jsonl.py diff --git a/src-py/renatoxsr/scripts/bulk_jsonl.py b/src-py/renatoxsr/scripts/bulk_jsonl.py new file mode 100644 index 0000000..8854efd --- /dev/null +++ b/src-py/renatoxsr/scripts/bulk_jsonl.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# vim: set ft=python ts=4 sw=4 et encoding=utf-8 : +# SPDX-FileCopyrightText: 2025-present RenatoXSR +# SPDX-License-Identifier: BSD-3-Clause-Clear +import argparse, json, os, sys +from pathlib import Path +import datetime as dt + +from renatoxsr.utils import hash_b58, pandoc +from renatoxsr.logger import set_logger +logger = set_logger(__name__) +if "--debug" in sys.argv: + logger.setLevel("DEBUG") + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--src", default=os.getcwd()) + p.add_argument("--glob", default="*") + p.add_argument("--overwrite", default=False) + p.add_argument("--logfile", default=sys.stderr, help="Log output(default: sys.stderr)") + p.add_argument("--loglevel", default="INFO", help="Log level (default: INFO)") + p.add_argument("filename", + default=dt.datetime.now().isoformat().replace(":", "_"), + help="output file (default: `datetime.now().isoformat()`)") + return p.parse_args() + +def main(): + parsed_args = parse_args() + logger.setLevel(parsed_args.loglevel) + + with open(Path(parsed_args.filename), "w" if parsed_args.overwrite else "a") as output_file: + for fpath in Path(parsed_args.src).glob(parsed_args.glob): + data = {"hash": hash_b58(fpath), "fpath": str(fpath), "markdown": pandoc(fpath)} + if not data["markdown"]: + logger.error("No markdown output from '%s' (hash:%s)", fpath, data["hash"]) + continue + output_file.write(json.dumps(data)+"\n") + + +if __name__ == "__main__": + main() \ No newline at end of file