Batch processing

from reducto import AsyncReducto
from pathlib import Path
import asyncio
from tqdm.asyncio import tqdm

client = AsyncReducto()

MAX_CONCURRENCY = 1000
FILES_TO_PARSE = list(Path("docs").glob("*.pdf"))


async def main():
    sem = asyncio.Semaphore(MAX_CONCURRENCY)

    async def parse_document(path: Path):
        async with sem:
            upload = await client.upload(file=path)
            result = await client.parse.run(document_url=upload)
            output_path = path.with_suffix(".reducto.json")
            output_path.write_text(result.model_dump_json())

    await tqdm.gather(
        *[parse_document(path) for path in FILES_TO_PARSE], desc="Parsing documents"
    )


if __name__ == "__main__":
    asyncio.run(main())

import Reducto from 'reductoai';
import fs from 'fs';
import { glob } from 'glob';

const client = new Reducto();
const FILES_TO_PARSE = glob.sync('docs/*.pdf');

async function main() {
  await Promise.all(
    FILES_TO_PARSE.map(async (file) => {
      console.log(`Processing ${file}`);
      const upload = await client.upload({ file: fs.createReadStream(file) });
      const result = await client.parse.run({ document_url: upload });
      const outputFile = file.replace(/\.pdf$/, '.reducto.json');
      await fs.promises.writeFile(outputFile, JSON.stringify(result, null, 2));
      console.log(`Finished processing ${file}`);
    })
  );
}

main().catch(console.error);