#!/usr/bin/env bash
# Purpose: build a small public-domain corpus from Project Gutenberg plain-text
#          eBooks: fetch each book once, strip the licence header and footer that
#          would otherwise be the most-repeated text in the corpus, and write a
#          provenance record so the corpus can be licensed, cited and rebuilt
# Platform: all (plain shell; needs curl and awk)
# Minimum memory: 8 GB
# Assumes: curl and awk are on PATH; the machine can reach www.gutenberg.org; you
#          have read https://www.gutenberg.org/policy/permission.html and checked
#          the copyright position of each title in your own country
set -euo pipefail

# ---------------------------------------------------------------- settings ---
# Book numbers from the Project Gutenberg catalogue. Replace these with the ones
# that make up your domain; the point of the project is that the corpus is yours.
BOOK_IDS="${BOOK_IDS:-1661 2852 108 244 834 3289}"
OUT_DIR="${OUT_DIR:-$PWD/domain-corpus}"
DELAY="${DELAY:-2}"     # seconds between requests; be a good guest
BASE_URL="${BASE_URL:-https://www.gutenberg.org/cache/epub}"

TEXT_DIR="$OUT_DIR/text"
RAW_DIR="$OUT_DIR/raw"
SOURCES="$OUT_DIR/sources.tsv"
TODAY="$(date -u +%Y-%m-%d)"

usage() {
    cat <<'USAGE'
Usage: [BOOK_IDS="1661 2852"] [OUT_DIR=./domain-corpus] bash fetch-gutenberg-corpus.sh

Downloads each Project Gutenberg book as plain UTF-8 text, keeps the raw file for
reference, writes a stripped copy with the licence header and footer removed, and
appends one tab-separated provenance row per book to sources.tsv.

Project Gutenberg's permission page states that the vast majority of its eBooks are
in the public domain in the United States, and also that "Not all items that are
public domain in the US are public domain in other countries, and vice versa."
Checking that for your own jurisdiction is your job, not this script's.
USAGE
}

command -v curl >/dev/null 2>&1 || { usage; echo "ERROR: curl is not on PATH." >&2; exit 1; }

mkdir -p "$TEXT_DIR" "$RAW_DIR"
if [ ! -f "$SOURCES" ]; then
    printf 'book_id\ttitle\turl\tretrieved\tbytes_raw\tbytes_stripped\n' > "$SOURCES"
fi

# ------------------------------------------------------------------ fetch ----
for id in $BOOK_IDS; do
    raw="$RAW_DIR/pg$id.txt"
    out="$TEXT_DIR/pg$id.txt"
    url="$BASE_URL/$id/pg$id.txt"

    if [ -s "$raw" ]; then
        echo "==> $id already downloaded"
    else
        echo "==> fetching $url"
        curl -fsSL --max-time 120 -o "$raw.part" "$url"
        mv "$raw.part" "$raw"
        sleep "$DELAY"
    fi

    # Every Project Gutenberg plain-text file wraps the work between two marker
    # lines. Everything outside them is boilerplate that is byte-identical across
    # books, which makes it the single most over-represented text in a corpus of
    # this size if it is left in.
    awk '
        /^\*\*\* *START OF (THE|THIS) PROJECT GUTENBERG EBOOK/ { inside = 1; next }
        /^\*\*\* *END OF (THE|THIS) PROJECT GUTENBERG EBOOK/   { inside = 0; next }
        inside { print }
    ' "$raw" > "$out"

    if [ ! -s "$out" ]; then
        echo "WARNING: no marker lines found in $raw; keeping the raw text instead." >&2
        echo "         Open it and check the format before training on it." >&2
        cp "$raw" "$out"
    fi

    title="$(grep -m1 '^Title:' "$raw" | sed 's/^Title:[[:space:]]*//' | tr -d '\r\t' || true)"
    [ -n "$title" ] || title="(title line not found)"
    raw_bytes="$(wc -c < "$raw" | tr -d ' ')"
    out_bytes="$(wc -c < "$out" | tr -d ' ')"

    printf '%s\t%s\t%s\t%s\t%s\t%s\n' \
        "$id" "$title" "$url" "$TODAY" "$raw_bytes" "$out_bytes" >> "$SOURCES"
    echo "    $title"
    echo "    kept $out_bytes of $raw_bytes bytes"
done

TOTAL_BYTES="$(cat "$TEXT_DIR"/*.txt | wc -c | tr -d ' ')"

cat <<SUMMARY

==> done
    books          $(find "$TEXT_DIR" -name '*.txt' | wc -l | tr -d ' ')
    stripped text  $TEXT_DIR
    raw text       $RAW_DIR
    provenance     $SOURCES
    total bytes    $TOTAL_BYTES

    A rough token count is bytes divided by the bytes-per-token ratio your
    tokeniser achieves, which tok_eval prints. Before training, read sources.tsv
    and satisfy yourself about the copyright position of every row in your own
    country; the Project Gutenberg permission page is the starting point and not
    the end of that question.

    Next: python make-domain-shards.py --input $TEXT_DIR
SUMMARY