{ "_id": "6a47d294e06c18f0a6520060", "Package": "crawlee", "Title": "Tidy Interface for Reproducible Web Crawling", "Version": "0.1.0", "Authors@R": "c(\nperson(\"Andre\", \"Leite\", email = \"leite@castlab.org\", role = c(\"aut\", \"cre\")),\nperson(\"Marcos\", \"Wasilew\", email = \"marcos.wasilew@gmail.com\", role = \"aut\"),\nperson(\"Hugo\", \"Vasconcelos\", email = \"hugo.vasconcelos@ufpe.br\", role = \"aut\"),\nperson(\"Carlos\", \"Amorin\", email = \"carlos.agaf@ufpe.br\", role = \"aut\"),\nperson(\"Diogo\", \"Bezerra\", email = \"diogo.bezerra@ufpe.br\", role = \"aut\"))", "Description": "A tidy, pipe-friendly toolkit for reproducible web\ncrawling and structured data collection, inspired by the\narchitecture of the 'Crawlee' library. Provides a unified\ncrawler with a deduplicating, resumable request queue,\ncontent-type aware handlers, structured storage backends and\nrich console logging via 'cli'. Supports crawling HTML pages,\nsitemaps, RSS and Atom feeds and PDF documents, with optional\nheadless-browser rendering and helpers for retrieval-augmented\ngeneration.", "License": "MIT + file LICENSE", "URL": "https://github.com/StrategicProjects/crawlee,\nhttps://strategicprojects.github.io/crawlee/", "BugReports": "https://github.com/StrategicProjects/crawlee/issues", "VignetteBuilder": "knitr", "Config/testthat/edition": "3", "Encoding": "UTF-8", "Language": "en-US", "RoxygenNote": "8.0.0", "NeedsCompilation": "no", "Packaged": { "Date": "2026-07-03 15:14:10 UTC", "User": "root" }, "Author": "Andre Leite [aut, cre], Marcos Wasilew [aut], Hugo Vasconcelos\n[aut], Carlos Amorin [aut], Diogo Bezerra [aut]", "Maintainer": "Andre Leite ", "Repository": "https://cran.r-universe.dev", "Date/Publication": "2026-07-03 12:10:02 UTC", "RemoteUrl": "https://github.com/cran/crawlee", "RemoteRef": "HEAD", "RemoteSha": "79ad61db592dc074abd5d4f556f810b2ebd69e18", "_user": "cran", "_type": "src", "_file": "crawlee_0.1.0.tar.gz", "_fileid": "https://r2.ropensci.org/07b667852eef7cc0cf5dcb5bdb85b1b1bcd9d992baaaed8b6075095fdc468997", "_filesize": 250791, "_sha256": "07b667852eef7cc0cf5dcb5bdb85b1b1bcd9d992baaaed8b6075095fdc468997", "_expires": "2026-10-11T15:17:38.000Z", "_created": "2026-07-03T15:14:10.000Z", "_published": "2026-07-03T15:17:40.140Z", "_jobs": [ { "job": 85028243688, "time": 156, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "8069952956" }, { "job": 85028243721, "time": 151, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8069951235" }, { "job": 85027578310, "time": 243, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "8069899432" }, { "job": 85028243623, "time": 163, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "8069955259" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/cran/actions/runs/28668961793", "_status": "success", "_upstream": "https://github.com/cran/crawlee", "_commit": { "id": "79ad61db592dc074abd5d4f556f810b2ebd69e18", "author": "Andre Leite ", "committer": "cran-robot ", "message": "version 0.1.0\n", "time": 1783080602 }, "_maintainer": { "name": "Andre Leite", "email": "leite@castlab.org" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.1.0", "role": "Depends" }, { "package": "cli", "role": "Imports" }, { "package": "httr2", "role": "Imports" }, { "package": "R6", "role": "Imports" }, { "package": "rlang", "role": "Imports" }, { "package": "rvest", "role": "Imports" }, { "package": "tibble", "role": "Imports" }, { "package": "vctrs", "role": "Imports" }, { "package": "xml2", "role": "Imports" }, { "package": "arrow", "role": "Suggests" }, { "package": "chromote", "role": "Suggests" }, { "package": "DBI", "role": "Suggests" }, { "package": "dplyr", "role": "Suggests" }, { "package": "duckdb", "role": "Suggests" }, { "package": "httptest2", "role": "Suggests" }, { "package": "jsonlite", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "later", "role": "Suggests" }, { "package": "nanoparquet", "role": "Suggests" }, { "package": "pdftools", "role": "Suggests" }, { "package": "polite", "role": "Suggests" }, { "package": "promises", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "cran", "_selfowned": false, "_usedby": 0, "_updates": [ { "week": "2026-27", "n": 1 } ], "_tags": [ { "name": "0.1.0", "date": "2026-07-03" } ], "_stars": 0, "_userbio": { "uuid": 6899542, "type": "organization", "name": "cran", "followers": 615, "description": "Unofficial read-only mirror of all CRAN R packages" }, "_downloads": { "count": 0, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/crawlee" }, "_devurl": "https://github.com/strategicprojects/crawlee", "_pkgdown": "https://strategicprojects.github.io/crawlee/", "_searchresults": 0, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/crawlee.html", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "LICENSE", "manual.pdf" ], "_cranurl": false, "_releases": [ { "version": "0.1.0", "date": "2026-07-03" } ], "_exports": [ "cr_autoscale", "cr_chunk", "cr_close", "cr_collect", "cr_dataset", "cr_embed", "cr_export", "cr_from_rss", "cr_from_sitemap", "cr_normalize_url", "cr_on_html", "cr_on_pdf", "cr_options", "cr_parallel", "cr_persist", "cr_run", "cr_store", "cr_stream", "cr_use_browser", "cr_use_http", "crawler", "Crawler", "Dataset", "KeyValueStore", "RequestQueue" ], "_help": [ { "page": "cr_autoscale", "title": "Enable autoscaled parallel fetching", "topics": [ "cr_autoscale" ] }, { "page": "cr_chunk", "title": "Chunk text for retrieval-augmented generation", "topics": [ "cr_chunk" ] }, { "page": "cr_close", "title": "Release a crawler's resources", "topics": [ "cr_close" ] }, { "page": "cr_collect", "title": "Collect crawl results", "topics": [ "cr_collect" ] }, { "page": "cr_dataset", "title": "Configure the dataset backend", "topics": [ "cr_dataset" ] }, { "page": "cr_embed", "title": "Attach embeddings to chunks", "topics": [ "cr_embed" ] }, { "page": "cr_export", "title": "Export chunks (and embeddings) for retrieval", "topics": [ "cr_export" ] }, { "page": "cr_from_rss", "title": "Discover URLs from an RSS or Atom feed", "topics": [ "cr_from_rss" ] }, { "page": "cr_from_sitemap", "title": "Discover URLs from a sitemap", "topics": [ "cr_from_sitemap" ] }, { "page": "cr_normalize_url", "title": "Normalise a URL into a canonical form", "topics": [ "cr_normalize_url" ] }, { "page": "cr_on_html", "title": "Register an HTML handler", "topics": [ "cr_on_html" ] }, { "page": "cr_on_pdf", "title": "Register a PDF handler", "topics": [ "cr_on_pdf" ] }, { "page": "cr_options", "title": "Set crawler options", "topics": [ "cr_options" ] }, { "page": "cr_parallel", "title": "Enable parallel (concurrent) fetching", "topics": [ "cr_parallel" ] }, { "page": "cr_persist", "title": "Persist a crawl to a run directory (and resume it)", "topics": [ "cr_persist" ] }, { "page": "cr_run", "title": "Run a crawl", "topics": [ "cr_run" ] }, { "page": "cr_store", "title": "Configure the key-value store for binary content", "topics": [ "cr_store" ] }, { "page": "cr_stream", "title": "Enable the streaming scheduler", "topics": [ "cr_stream" ] }, { "page": "cr_use_browser", "title": "Use the headless-browser fetch backend", "topics": [ "cr_use_browser" ] }, { "page": "cr_use_http", "title": "Use the HTTP fetch backend", "topics": [ "cr_use_http" ] }, { "page": "crawler", "title": "Create a crawler", "topics": [ "crawler" ] }, { "page": "Crawler-class", "title": "Crawler", "topics": [ "Crawler", "Crawler-class" ] }, { "page": "Dataset", "title": "Dataset", "topics": [ "Dataset" ] }, { "page": "KeyValueStore", "title": "Key-value store", "topics": [ "KeyValueStore" ] }, { "page": "RequestQueue", "title": "Request queue", "topics": [ "RequestQueue" ] } ], "_pkglogo": "https://github.com/cran/crawlee/raw/HEAD/man/figures/logo.svg", "_readme": "https://github.com/cran/crawlee/raw/HEAD/README.md", "_rundeps": [ "askpass", "cli", "curl", "glue", "httr", "httr2", "jsonlite", "lifecycle", "magrittr", "mime", "openssl", "pillar", "pkgconfig", "R6", "rappdirs", "rlang", "rvest", "selectr", "sys", "tibble", "utf8", "vctrs", "withr", "xml2" ], "_vignettes": [ { "source": "rag-pipeline.Rmd", "filename": "rag-pipeline.html", "title": "A RAG pipeline", "engine": "knitr::rmarkdown", "headings": [ "1. Crawl and collect text", "2. Chunk", "3. Embed", "4. Export for retrieval", "End to end" ], "created": "2026-07-03 12:10:02", "modified": "2026-07-03 12:10:02", "commits": 1 }, { "source": "crawling-a-website.Rmd", "filename": "crawling-a-website.html", "title": "Crawling a website", "engine": "knitr::rmarkdown", "headings": [ "The model", "Your first crawler", "Following links", "Controlling scope", "Routing different page types", "Crawling from a sitemap", "Rendering JavaScript pages", "Where next" ], "created": "2026-07-03 12:10:02", "modified": "2026-07-03 12:10:02", "commits": 1 }, { "source": "crawlee.Rmd", "filename": "crawlee.html", "title": "Getting started with crawlee", "engine": "knitr::rmarkdown", "headings": [ "The mental model", "A minimal crawl", "The handler context", "Controlling link discovery", "Reproducibility" ], "created": "2026-07-03 12:10:02", "modified": "2026-07-03 12:10:02", "commits": 1 }, { "source": "scaling-and-politeness.Rmd", "filename": "scaling-and-politeness.html", "title": "Scaling and politeness", "engine": "knitr::rmarkdown", "headings": [ "Being a good web citizen", "Going faster", "Fixed-concurrency batches — cr_parallel()", "Adaptive batches — cr_autoscale()", "Continuous streaming pool — cr_stream()", "Choosing an engine", "Combining with persistence" ], "created": "2026-07-03 12:10:02", "modified": "2026-07-03 12:10:02", "commits": 1 }, { "source": "storage-and-resumable-runs.Rmd", "filename": "storage-and-resumable-runs.html", "title": "Storage and resumable runs", "engine": "knitr::rmarkdown", "headings": [ "The dataset", "The key-value store", "The request queue and reproducibility", "One-call setup: cr_persist()", "Resuming" ], "created": "2026-07-03 12:10:02", "modified": "2026-07-03 12:10:02", "commits": 1 } ], "_score": 3.3979400086720375, "_indexed": false, "_nocasepkg": "crawlee", "_universes": [ "cran", "strategicprojects" ], "_indexurl": "https://strategicprojects.r-universe.dev/crawlee", "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.0", "date": "2026-07-03T15:16:46.000Z", "distro": "resolute", "commit": "79ad61db592dc074abd5d4f556f810b2ebd69e18", "fileid": "https://r2.ropensci.org/38f916735a8fe64cb529639ac621440e691b32ef84cc6fee6fd04d55731eedcf", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/cran/actions/runs/28668961793" }, { "r": "4.6.1", "os": "linux", "version": "0.1.0", "date": "2026-07-03T15:16:39.000Z", "distro": "resolute", "commit": "79ad61db592dc074abd5d4f556f810b2ebd69e18", "fileid": "https://r2.ropensci.org/bb2acd62ad74c84916af5a9d312ca673050908d0a86b84b0282fc91738a3a6da", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/cran/actions/runs/28668961793" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.0", "date": "2026-07-03T15:17:15.000Z", "commit": "79ad61db592dc074abd5d4f556f810b2ebd69e18", "fileid": "https://r2.ropensci.org/231a924f34330ec41d566842577df363098accd813cbeefaaeb7a67c8d51c8d1", "status": "success", "buildurl": "https://github.com/r-universe/cran/actions/runs/28668961793" } ] }