Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions samples/pyodide-fastapi/config.capnp

This file was deleted.

57 changes: 0 additions & 57 deletions samples/pyodide-fastapi/worker.py

This file was deleted.

28 changes: 0 additions & 28 deletions samples/pyodide-langchain/config.capnp

This file was deleted.

15 changes: 0 additions & 15 deletions samples/pyodide-langchain/worker.py

This file was deleted.

32 changes: 3 additions & 29 deletions src/pyodide/internal/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
SHOULD_SNAPSHOT_TO_DISK,
IS_CREATING_BASELINE_SNAPSHOT,
MEMORY_SNAPSHOT_READER,
REQUIREMENTS,
IS_CREATING_SNAPSHOT,
IS_EW_VALIDATING,
IS_DYNAMIC_WORKER,
Expand Down Expand Up @@ -453,7 +452,7 @@ function recordDsoHandles(Module: Module): DsoHandles {
*
* This function returns a list of modules that have been imported.
*/
function memorySnapshotDoImports(Module: Module): string[] {
function memorySnapshotDoImports(Module: Module): void {
const baselineSnapshotImports =
MetadataReader.constructor.getBaselineSnapshotImports();
const toImport = baselineSnapshotImports.join(',');
Expand All @@ -465,31 +464,6 @@ function memorySnapshotDoImports(Module: Module): string[] {
simpleRunPython(Module, 'sysconfig.get_config_vars()');
// Delete to avoid polluting globals
simpleRunPython(Module, `del ${toDelete}`);
if (IS_CREATING_BASELINE_SNAPSHOT) {
// We've done all the imports for the baseline snapshot.
return [];
}
if (REQUIREMENTS.length == 0) {
// Don't attempt to scan for package imports if the Worker has specified no package
// requirements, as this means their code isn't going to be importing any modules that we need
// to include in a snapshot.
return [];
}
Comment thread
hoodmane marked this conversation as resolved.

// The `importedModules` list will contain all modules that have been imported, including local
// modules, the usual `js` and other stdlib modules. We want to filter out local imports, so we
// grab them and put them into a set for fast filtering.
const importedModules: string[] = MetadataReader.getPackageSnapshotImports(
Module.API.version
);
const deduplicatedModules = [...new Set(importedModules)];

// Import the modules list so they are included in the snapshot.
if (deduplicatedModules.length > 0) {
simpleRunPython(Module, 'import ' + deduplicatedModules.join(','));
}

return deduplicatedModules;
}

function describeValue(val: any): string {
Expand Down Expand Up @@ -903,7 +877,7 @@ export function maybeCollectSnapshot(
): void {
// In order to surface any problems that occur in `memorySnapshotDoImports` to
// users in local development, always call it even if we aren't actually
const importedModulesList = memorySnapshotDoImports(Module);
memorySnapshotDoImports(Module);
if (!IS_CREATING_SNAPSHOT) {
return;
}
Expand All @@ -916,7 +890,7 @@ export function maybeCollectSnapshot(

collectSnapshot(
Module,
importedModulesList,
[],
Comment thread
ryanking13 marked this conversation as resolved.
customSerializedObjects,
IS_CREATING_BASELINE_SNAPSHOT ? 'baseline' : 'package'
);
Expand Down
2 changes: 1 addition & 1 deletion src/pyodide/make_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def make_config(
) -> str:
requirements = ""
for name in reqs:
requirements += f'(name="{name}", pythonRequirement=""),'
requirements += f'(name="{name}", obsoletePythonRequirement=""),'

compat_flags = ""
for flag in flags:
Expand Down
4 changes: 2 additions & 2 deletions src/pyodide/types/runtime-generated/metadata.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ declare namespace MetadataReader {
const isTracing: () => boolean;
const shouldSnapshotToDisk: () => boolean;
const isCreatingBaselineSnapshot: () => boolean;
const getRequirements: () => string[];
const getMainModule: () => string;
const hasMemorySnapshot: () => boolean;
const getNames: () => string[];
Expand All @@ -33,8 +32,9 @@ declare namespace MetadataReader {
const getPyodideVersion: () => string;
const getPackagesVersion: () => string;
const getPackagesLock: () => string;
const read: (index: number, position: number, buffer: Uint8Array) => number;
const getRequirements: () => string[];
const getTransitiveRequirements: () => Set<string>;
const read: (index: number, position: number, buffer: Uint8Array) => number;
const getCompatibilityFlags: () => CompatibilityFlags;
const setCpuLimitNearlyExceededCallback: (
buf: Uint8Array,
Expand Down
2 changes: 1 addition & 1 deletion src/workerd/io/bundle-fs.c++
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ kj::Rc<Directory> getBundleDirectory(const WorkerSource& conf) {
.data = pythonModule.body.asBytes(),
});
}
KJ_CASE_ONEOF(pythonRequirement, WorkerSource::PythonRequirement) {
KJ_CASE_ONEOF(pythonRequirement, WorkerSource::ObsoletePythonRequirement) {
// Just ignore it.
}
KJ_CASE_ONEOF(capnpModule, WorkerSource::CapnpModule) {
Expand Down
4 changes: 2 additions & 2 deletions src/workerd/io/worker-modules.c++
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ kj::Own<api::pyodide::PyodideMetadataReader::State> createPyodideMetadataState(
KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
numFiles++;
}
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
KJ_CASE_ONEOF(content, Worker::Script::ObsoletePythonRequirement) {
numRequirements++;
}
KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
Expand Down Expand Up @@ -77,7 +77,7 @@ kj::Own<api::pyodide::PyodideMetadataReader::State> createPyodideMetadataState(
names.add(kj::str(module.name));
contents.add(kj::heapArray(content.body.asBytes()));
}
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
KJ_CASE_ONEOF(content, Worker::Script::ObsoletePythonRequirement) {
requirements.add(kj::str(module.name));
}
KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
Expand Down
6 changes: 3 additions & 3 deletions src/workerd/io/worker-modules.h
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ static kj::Arc<jsg::modules::ModuleRegistry> newWorkerModuleRegistry(
// bundleBuilder.addEsmModule(def.name, entry);
// break;
}
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
KJ_CASE_ONEOF(content, Worker::Script::ObsoletePythonRequirement) {
// Handled separately
break;
}
Expand Down Expand Up @@ -362,7 +362,7 @@ kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileLegacyModule(jsg::Lock& js,
// Nothing to do. Handled elsewhere.
return kj::none;
}
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
KJ_CASE_ONEOF(content, Worker::Script::ObsoletePythonRequirement) {
// Nothing to do. Handled elsewhere.
return kj::none;
}
Expand Down Expand Up @@ -416,7 +416,7 @@ kj::Array<Worker::Script::CompiledGlobal> compileServiceWorkerGlobals(jsg::Lock&
KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
KJ_FAIL_REQUIRE("modules not supported with mainScript");
}
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
KJ_CASE_ONEOF(content, Worker::Script::ObsoletePythonRequirement) {
KJ_FAIL_REQUIRE("modules not supported with mainScript");
}
KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
Expand Down
9 changes: 4 additions & 5 deletions src/workerd/io/worker-source.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,8 @@ struct WorkerSource {
kj::StringPtr body;
};

// PythonRequirement is a variant of ModuleContent, but has no body. The module name specifies
// a Python package to be provided by the system.
struct PythonRequirement {};
// This is no longer supported by Python, but it used to define built-in packages.
Comment thread
ryanking13 marked this conversation as resolved.
struct ObsoletePythonRequirement {};

// CapnpModule is a .capnp Cap'n Proto schema file. The original text of the file isn't provided;
// instead, `ModulesSource::capnpSchemas` contains all the capnp schemas needed by the Worker,
Expand All @@ -77,7 +76,7 @@ struct WorkerSource {
WasmModule,
JsonModule,
PythonModule,
PythonRequirement,
ObsoletePythonRequirement,
CapnpModule>;

struct Module {
Expand Down Expand Up @@ -116,7 +115,7 @@ struct WorkerSource {
KJ_CASE_ONEOF(content, PythonModule) {
result.content = content;
}
KJ_CASE_ONEOF(content, PythonRequirement) {
KJ_CASE_ONEOF(content, ObsoletePythonRequirement) {
result.content = content;
}
KJ_CASE_ONEOF(content, CapnpModule) {
Expand Down
2 changes: 1 addition & 1 deletion src/workerd/io/worker.h
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ class Worker::Script: public kj::AtomicRefcounted {
using WasmModule = WorkerSource::WasmModule;
using JsonModule = WorkerSource::JsonModule;
using PythonModule = WorkerSource::PythonModule;
using PythonRequirement = WorkerSource::PythonRequirement;
using ObsoletePythonRequirement = WorkerSource::ObsoletePythonRequirement;
using CapnpModule = WorkerSource::CapnpModule;
using ModuleContent = WorkerSource::ModuleContent;
using Module = WorkerSource::Module;
Expand Down
5 changes: 2 additions & 3 deletions src/workerd/server/server.c++
Original file line number Diff line number Diff line change
Expand Up @@ -5834,11 +5834,10 @@ kj::Promise<void> Server::preloadPython(
// Preload Python packages.
KJ_IF_SOME(modulesSource, workerDef.source.variant.tryGet<Worker::Script::ModulesSource>()) {
if (modulesSource.isPython) {
auto pythonRequirements = getPythonRequirements(modulesSource);

// Store the packages in the package manager that is stored in the pythonConfig
co_await server::fetchPyodidePackages(pythonConfig, pythonConfig.pyodidePackageManager,
pythonRequirements, release, network, timer);
co_await server::fetchPyodidePackages(
pythonConfig, pythonConfig.pyodidePackageManager, {}, release, network, timer);
Comment on lines +5839 to +5840
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need fetchPyodidePackages anymore?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seemed like it, I started getting errors about missing crypto when I removed it.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment?

}
}
}
Expand Down
19 changes: 0 additions & 19 deletions src/workerd/server/tests/python/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
load("//src/workerd/server/tests/python:import_tests.bzl", "gen_rust_import_tests")
load("//src/workerd/server/tests/python:py_wd_test.bzl", "py_wd_test", "python_test_setup")
load("//src/workerd/server/tests/python/vendor_pkg_tests:vendor_test.bzl", "vendored_py_wd_test")

Expand Down Expand Up @@ -36,16 +35,10 @@ py_wd_test(
compat_date = "2026-01-01",
)

gen_rust_import_tests()

py_wd_test("undefined-handler")

py_wd_test("vendor_dir")

py_wd_test("dont-snapshot-pyodide")

py_wd_test("filter-non-py-files")

py_wd_test("durable-object")

py_wd_test(
Expand All @@ -70,16 +63,4 @@ py_wd_test("vendor_dir_compat_flag")

py_wd_test("default-class-with-legacy-global-handlers")

py_wd_test(
"fastapi",
make_snapshot = False,
use_snapshot = "fastapi",
)

py_wd_test(
"numpy",
make_snapshot = False,
use_snapshot = "numpy",
)

py_wd_test("python-compat-flag")

This file was deleted.

Loading
Loading