NLTK downloads added to dockerfile

Author: Peter Johnson

Dockerfile

@@ -20,12 +20,18 @@ RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
     find /app/.venv -path "*/tests/*" -delete && \
     find /app/.venv -path "*/test/*" -delete && \
     find /app/.venv -name "*.md" -delete && \
-    find /app/.venv -name "*.txt" -delete
+    find /app/.venv -name "*.txt" -delete &&\
+    # Download NLTK corpora into the venv so it's cached ---
+    /app/.venv/bin/python -m nltk.downloader -d /app/.venv/nltk_data brown reuters gutenberg webtext
+
+ENV NLTK_DATA=/app/.venv/nltk_data
 
 FROM ghcr.io/lambda-feedback/evaluation-function-base/python:3.12
 
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
+    ENV NLTK_DATA=/app/.venv/nltk_data
+
 
 # Copy the cleaned virtual environment
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}

evaluation_function/models/shannon_words_ngram.py

@@ -1,17 +1,16 @@
 """
 A simple n-gram (word) Shannon-style language model with add-one smoothing.
 """
-from lf_toolkit.evaluation import Result, Params
-import random, pickle, os
-import os
-import tempfile
+import sys, traceback, os
+import random, pickle, tempfile, re
 from pathlib import Path
 from io import StringIO
-import re
+from lf_toolkit.evaluation import Result, Params
 from .utils import csv_to_lists
 import nltk
 from nltk.corpus import brown, reuters, gutenberg, webtext
 
+# Local users run the following once (no need if using Docker):
 #nltk.download("brown"); nltk.download("reuters"); nltk.download("gutenberg"); nltk.download("webtext")  # CHANGE (one-time)
 
 START, END = "<s>", "</s>"