{ "_id": "6ace2b71a14063b82c3b0b9b", "Package": "piecemaker", "Title": "Tools for Preparing Text for Tokenizers", "Version": "1.0.2.9000", "Authors@R": "c(\nperson(\"Jon\", \"Harmon\", , \"jonthegeek@gmail.com\", role = c(\"aut\", \"cre\"),\ncomment = c(ORCID = \"0000-0003-4781-4346\")),\nperson(\"Jonathan\", \"Bratt\", , \"jonathan.bratt@macmillan.com\", role = \"aut\",\ncomment = c(ORCID = \"0000-0003-2859-0076\")),\nperson(\"Bedford Freeman & Worth Pub Grp LLC DBA Macmillan Learning\", role = \"cph\")\n)", "Description": "Tokenizers break text into pieces that are more usable by\nmachine learning models. Many tokenizers share some preparation\nsteps. This package provides those shared steps, along with a\nsimple tokenizer.", "License": "Apache License (>= 2)", "URL": "https://github.com/macmillancontentscience/piecemaker,\nhttps://macmillancontentscience.github.io/piecemaker/", "BugReports": "https://github.com/macmillancontentscience/piecemaker/issues", "Config/testthat/edition": "3", "Encoding": "UTF-8", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.2.3", "Config/pak/sysreqs": "libicu-dev", "Repository": "https://macmillancontentscience.r-universe.dev", "Date/Publication": "2023-06-02 19:46:08 UTC", "RemoteUrl": "https://github.com/macmillancontentscience/piecemaker", "RemoteRef": "HEAD", "RemoteSha": "b02c1a74923301545366805680e54091675305c6", "NeedsCompilation": "no", "Packaged": { "Date": "2026-10-13 12:25:03 UTC", "User": "root" }, "Author": "Jon Harmon [aut, cre] (ORCID: ),\nJonathan Bratt [aut] (ORCID: ),\nBedford Freeman & Worth Pub Grp LLC DBA Macmillan Learning [cph]", "Maintainer": "Jon Harmon ", "_user": "macmillancontentscience", "_type": "src", "_file": "piecemaker_1.0.2.9000.tar.gz", "_fileid": "https://r2.ropensci.org/3609bd06e0e867acd6f3b795e8f326fcc784acf1fda58b04fa51a75bba39c793", "_filesize": 114914, "_sha256": "3609bd06e0e867acd6f3b795e8f326fcc784acf1fda58b04fa51a75bba39c793", "_expires": "2027-01-21T13:00:31.000Z", "_created": "2026-10-13T12:25:03.000Z", "_published": "2026-10-13T13:00:33.325Z", "_buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039", "_status": "success", "_host": "GitHub-Actions", "_jobs": [ { "job": 115295210496, "time": 139, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11812558201" }, { "job": 115295210266, "time": 105, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11812388325" }, { "job": 115295210375, "time": 77, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "11814268801" }, { "job": 115295210577, "time": 106, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11816305531" }, { "job": 115293795187, "time": 203, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "11812382892" }, { "job": 115295210521, "time": 91, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "11812940706" }, { "job": 115295210701, "time": 58, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11813020546" }, { "job": 115295210895, "time": 60, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "11812628039" }, { "job": 115295210523, "time": 60, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11812378190" } ], "_upstream": "https://github.com/macmillancontentscience/piecemaker", "_commit": { "id": "b02c1a74923301545366805680e54091675305c6", "author": "Jon Harmon ", "committer": "Jon Harmon ", "message": "Increment version number to 1.0.2.9000\n", "time": 1685735168 }, "_maintainer": { "name": "Jon Harmon", "email": "jonthegeek@gmail.com", "login": "jonthegeek", "mastodon": "@jonthegeek@fosstodon.org", "bluesky": "@jonthegeek.com", "linkedin": "in/jonthegeek", "description": "Executive Director at DSLC.io (fka @r4ds) | Principal Developer (@atorus-research)", "uuid": 33983824, "orcid": "0000-0003-4781-4346" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 2.10", "role": "Depends" }, { "package": "cli", "role": "Imports" }, { "package": "glue", "role": "Imports" }, { "package": "rlang", "version": ">= 0.4.2", "role": "Imports" }, { "package": "stringi", "role": "Imports" }, { "package": "stringr", "role": "Imports" }, { "package": "covr", "role": "Suggests" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "macmillancontentscience", "_selfowned": true, "_usedby": 3, "_updates": [], "_tags": [], "_stars": 0, "_contributors": [ { "user": "jonthegeek", "count": 20, "uuid": 33983824 }, { "user": "jonathanbratt", "count": 1, "uuid": 33073024 } ], "_userbio": { "uuid": 73667768, "type": "organization", "name": "Macmillan Learning Content Science", "followers": 5, "description": "A team of data scientists researching and applying content analysis techniques" }, "_downloads": { "count": 497, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/piecemaker" }, "_devurl": "https://github.com/macmillancontentscience/piecemaker", "_pkgdown": "https://macmillancontentscience.github.io/piecemaker/", "_searchresults": 2, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/piecemaker.html", "extra/readme.html", "extra/readme.md" ], "_homeurl": "https://github.com/macmillancontentscience/piecemaker", "_realowner": "macmillancontentscience", "_cranurl": true, "_releases": [ { "version": "1.0.0", "date": "2021-08-06" }, { "version": "1.0.1", "date": "2022-03-03" }, { "version": "1.0.2", "date": "2023-06-02" } ], "_exports": [ "prepare_and_tokenize", "prepare_text", "remove_control_characters", "remove_diacritics", "remove_replacement_characters", "space_cjk", "space_punctuation", "squish_whitespace", "tokenize_space", "validate_utf8" ], "_help": [ { "page": "prepare_and_tokenize", "title": "Split Text on Spaces", "topics": [ "prepare_and_tokenize" ] }, { "page": "prepare_text", "title": "Prepare Text for Tokenization", "topics": [ "prepare_text" ] }, { "page": "remove_control_characters", "title": "Remove Non-Character Characters", "topics": [ "remove_control_characters" ] }, { "page": "remove_diacritics", "title": "Remove Diacritical Marks on Characters", "topics": [ "remove_diacritics" ] }, { "page": "remove_replacement_characters", "title": "Remove the Unicode Replacement Character", "topics": [ "remove_replacement_characters" ] }, { "page": "space_cjk", "title": "Add Spaces Around CJK Ideographs", "topics": [ "space_cjk" ] }, { "page": "space_punctuation", "title": "Add Spaces Around Punctuation", "topics": [ "space_punctuation" ] }, { "page": "squish_whitespace", "title": "Remove Extra Whitespace", "topics": [ "squish_whitespace" ] }, { "page": "tokenize_space", "title": "Break Text at Spaces", "topics": [ "tokenize_space" ] }, { "page": "validate_utf8", "title": "Clean Up Text to UTF-8", "topics": [ "validate_utf8" ] } ], "_readme": "https://github.com/macmillancontentscience/piecemaker/raw/HEAD/README.md", "_rundeps": [ "cli", "glue", "lifecycle", "magrittr", "rlang", "stringi", "stringr", "vctrs" ], "_score": 3.6532125137753435, "_indexed": true, "_nocasepkg": "piecemaker", "_universes": [ "macmillancontentscience", "jonthegeek" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "1.0.2.9000", "date": "2026-10-13T12:27:34.000Z", "distro": "resolute", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/69a6943e65934926d540e01c04f99ff2bff0ebfa51ba1a59414dd59746d6a1da", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.6.1", "os": "linux", "version": "1.0.2.9000", "date": "2026-10-13T12:26:57.000Z", "distro": "resolute", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/4f2fe576bcdc08724d5a306a031b06a1b7bcabdc49a6f4e38673d2d9d3746b39", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.5.3", "os": "mac", "version": "1.0.2.9000", "date": "2026-10-13T12:49:43.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/9534382cff9c725269c544395e804b212f28c1c3be7b6abe6686fbec775f3612", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.6.1", "os": "mac", "version": "1.0.2.9000", "date": "2026-10-13T12:59:46.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/afc64f5700403b41ab3cee941801c5c1cdcf99f7867ac864770fd77947208359", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.6.0", "os": "wasm", "version": "1.0.2.9000", "date": "2026-10-13T12:26:52.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/f803977fc4e6b598097420d6f503bcdc3e1457d61c4e57a05c5b3c085c65e550", "status": "success", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.7.0", "os": "win", "version": "1.0.2.9000", "date": "2026-10-13T12:26:05.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/fed2ef831bf973471b3b6a065d28d8664fd0a4f9995f09906fa4d77dcb8f155e", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.5.3", "os": "win", "version": "1.0.2.9000", "date": "2026-10-13T12:26:09.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/7d31c10c81ef23e66c76ab60d60e449000ec753511dcc7a72be3fc29d13d867d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" }, { "r": "4.6.1", "os": "win", "version": "1.0.2.9000", "date": "2026-10-13T12:26:07.000Z", "commit": "b02c1a74923301545366805680e54091675305c6", "fileid": "https://r2.ropensci.org/0efe048ec1d084355a36d1f5586ebbbd5a55e66e1da0516b2f43d2726f33be8c", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/macmillancontentscience/actions/runs/38396702039" } ] }