feat: cap directory listing at max_directory_count to avoid full scans

fileglancer/filestore.py

@@ -3,6 +3,7 @@
 rooted at a specific directory.
 """
 
+import itertools
 import os
 import stat
 try:
@@ -490,7 +491,8 @@ def check_is_binary(self, path: Optional[str] = None, sample_size: int = 4096) -
 
     def yield_file_infos_paginated(self, path: Optional[str] = None, current_user: str = None,
                                     session = None, limit: int = 200,
-                                    cursor: Optional[str] = None) -> tuple[list[FileInfo], bool, Optional[str], int]:
+                                    cursor: Optional[str] = None,
+                                    *, max_count: int) -> tuple[list[FileInfo], bool, Optional[str], int, bool]:
         """
         Return a page of FileInfo objects for children of the given path.
 
@@ -505,19 +507,33 @@ def yield_file_infos_paginated(self, path: Optional[str] = None, current_user: s
             limit: Maximum number of entries to return.
             cursor: Name of the last entry from the previous page. Entries after
                 this name (in sort order) are returned.
+            max_count: Maximum number of directory entries to read and sort.
+                Directories with more entries are truncated at this limit;
+                files beyond max_count are not accessible via pagination.
+                Required keyword arg — callers must pass the configured limit
+                (e.g. Settings.max_directory_count) so the advertised and
+                actual limits cannot drift.
 
         Returns:
-            Tuple of (file_infos, has_more, next_cursor, total_count).
+            Tuple of (file_infos, has_more, next_cursor, total_count, is_truncated).
+            total_count is capped at max_count. is_truncated is True when the
+            directory has more entries than max_count.
         """
         full_path = self._check_path_in_root(path)
 
         # Compute user groups once for the entire listing
         user_groups = FileInfo._get_user_groups(current_user) if current_user else None
 
-        # Collect and sort all entries — scandir's is_dir() is free on Linux
-        entries = list(os.scandir(full_path))
-        entries.sort(key=lambda e: (not e.is_dir(follow_symlinks=False), e.name))
+        # Read max_count + 1 entries: the extra one detects truncation without
+        # reading the entire directory.
+        with os.scandir(full_path) as scanner:
+            entries = list(itertools.islice(scanner, max_count + 1))
         total_count = len(entries)
+        is_truncated = total_count > max_count
+        if is_truncated:
+            entries = entries[:max_count]
+            total_count = max_count
+        entries.sort(key=lambda e: (not e.is_dir(follow_symlinks=False), e.name))
 
         # Apply cursor: skip past the cursor entry
         if cursor:
@@ -548,7 +564,7 @@ def yield_file_infos_paginated(self, path: Optional[str] = None, current_user: s
                 continue
 
         next_cursor = page_entries[-1].name if has_more and page_entries else None
-        return file_infos, has_more, next_cursor, total_count
+        return file_infos, has_more, next_cursor, total_count, is_truncated
 
     def yield_file_infos(self, path: Optional[str] = None, current_user: str = None, session = None) -> Generator[FileInfo, None, None]:
         """

fileglancer/server.py

@@ -1398,14 +1398,17 @@ async def get_file_metadata(path_name: str, subpath: Optional[str] = Query(''),
                     if file_info.is_dir:
                         try:
                             if limit is not None:
-                                files, has_more, next_cursor, total_count = filestore.yield_file_infos_paginated(
+                                files, has_more, next_cursor, total_count, is_truncated = filestore.yield_file_infos_paginated(
                                     subpath, current_user=username, session=session,
-                                    limit=limit, cursor=cursor
+                                    limit=limit, cursor=cursor,
+                                    max_count=settings.max_directory_count,
                                 )
                                 result["files"] = [json.loads(f.model_dump_json()) for f in files]
                                 result["has_more"] = has_more
                                 result["next_cursor"] = next_cursor
                                 result["total_count"] = total_count
+                                result["is_truncated"] = is_truncated
+                                result["max_count"] = settings.max_directory_count
                             else:
                                 files = list(filestore.yield_file_infos(subpath, current_user=username, session=session))
                                 result["files"] = [json.loads(f.model_dump_json()) for f in files]

fileglancer/settings.py

@@ -78,6 +78,10 @@ class Settings(BaseSettings):
     # Maximum size of the sharing key LRU cache
     sharing_key_cache_size: int = 1000
 
+    # Maximum number of directory entries reported in total_count for paginated listings.
+    # Prevents a full directory scan for the count in very large directories.
+    max_directory_count: int = 10000
+
     # OKTA OAuth/OIDC settings for authentication
     okta_domain: Optional[str] = None
     okta_client_id: Optional[str] = None
@@ -127,6 +131,13 @@ def validate_external_proxy_url(cls, v):
         if v is None or (isinstance(v, str) and v.strip() == ''):
             raise ValueError("Add external_proxy_url to your config.yaml or FGC_EXTERNAL_PROXY_URL to your .env file")
         return v
+
+    @field_validator('max_directory_count')
+    @classmethod
+    def validate_max_directory_count(cls, v: int) -> int:
+        if v <= 0:
+            raise ValueError('max_directory_count must be a positive integer')
+        return v
 
     @classmethod
     def settings_customise_sources(  # noqa: PLR0913

frontend/src/components/ui/BrowsePage/FileTable.tsx

@@ -22,6 +22,10 @@ import { makeBrowseLink } from '@/utils/index';
 import FgTooltip from '@/components/ui/widgets/FgTooltip';
 import { FgStyledLink } from '@/components/ui/widgets/FgLink';
 import { SortIcons } from '@/components/ui/Table/TableCard';
+import {
+  NotificationItem,
+  getNotificationStyles
+} from '@/components/ui/Notifications/NotificationItem';
 import {
   typeColumn,
   lastModifiedColumn,
@@ -96,7 +100,14 @@ export default function Table({
       parent = parent.parentElement;
     }
   }, []);
-  const sortingEnabled = !hasNextPage;
+  const sortingDisabledByTruncation = Boolean(fileQuery.data?.isTruncated);
+  const sortingDisabledByPagination =
+    !sortingDisabledByTruncation && Boolean(hasNextPage);
+  const sortingEnabled =
+    !sortingDisabledByTruncation && !sortingDisabledByPagination;
+  const sortingDisabledTooltip = sortingDisabledByTruncation
+    ? 'Sort unavailable: folder is too large and contents are truncated'
+    : 'Sort unavailable until all files are loaded';
 
   const selectedFileNames = useMemo(
     () => new Set(fileBrowserState.selectedFiles.map(file => file.name)),
@@ -312,8 +323,28 @@ export default function Table({
     navigate(link);
   });
 
+  const isTruncated = fileQuery.data?.isTruncated ?? false;
+  const maxCount = fileQuery.data?.maxCount ?? null;
+  const truncationLimit =
+    maxCount?.toLocaleString() ?? 'the configured limit of';
+
   return (
     <div className="min-w-full bg-background select-none" ref={tableRef}>
+      {isTruncated ? (
+        <div
+          className={`${getNotificationStyles('warning').container} p-4 rounded-md mb-3`}
+        >
+          <NotificationItem
+            notification={{
+              id: 0,
+              type: 'warning',
+              title: 'Folder contents truncated',
+              message: `This folder contains more than ${truncationLimit} items. Only ${truncationLimit} items are shown.`
+            }}
+            showDismissButton={false}
+          />
+        </div>
+      ) : null}
       <div className="bg-background border-b border-surface">
         {table.getHeaderGroups().map(headerGroup => (
           <div
@@ -349,7 +380,7 @@ export default function Table({
                       ) : (
                         <FgTooltip
                           icon={HiOutlineSwitchVertical}
-                          label="Sort unavailable until all files are loaded via infinite scroll"
+                          label={sortingDisabledTooltip}
                           triggerClasses="flex items-center opacity-40 cursor-not-allowed"
                         />
                       )

frontend/src/queries/fileQueries.ts

@@ -28,6 +28,8 @@ type FileBrowserPageResponse = {
   has_more?: boolean;
   next_cursor?: string | null;
   total_count?: number | null;
+  is_truncated?: boolean;
+  max_count?: number;
 };
 
 export type FileQueryData = {
@@ -37,6 +39,8 @@ export type FileQueryData = {
   errorMessage?: string;
   hasMore: boolean;
   totalCount: number | null;
+  isTruncated: boolean;
+  maxCount: number | null;
 };
 
 // Query key factory for hierarchical cache management
@@ -142,7 +146,9 @@ export default function useFileQuery(
       currentFileOrFolder,
       files: allFiles,
       hasMore: lastPage?.has_more ?? false,
-      totalCount
+      totalCount,
+      isTruncated: lastPage?.is_truncated ?? false,
+      maxCount: lastPage?.max_count ?? null
     };
   };