Add tests, move file reading to only be on demand for memory mapped provider, update docs

Author: HarelM

ElevationWebApi.sln

@@ -9,6 +9,8 @@ ProjectSection(SolutionItems) = preProject
 	Dockerfile = Dockerfile
 EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ElevationWebApiTests", "ElevationWebApiTests\ElevationWebApiTests.csproj", "{6210339D-AAD6-4BFC-9314-4C8A4FADFF67}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -19,5 +21,9 @@ Global
 		{DFEF0265-106B-4E81-A763-D2DE297E1AD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DFEF0265-106B-4E81-A763-D2DE297E1AD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DFEF0265-106B-4E81-A763-D2DE297E1AD3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6210339D-AAD6-4BFC-9314-4C8A4FADFF67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6210339D-AAD6-4BFC-9314-4C8A4FADFF67}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6210339D-AAD6-4BFC-9314-4C8A4FADFF67}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6210339D-AAD6-4BFC-9314-4C8A4FADFF67}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

ElevationWebApi.sln.DotSettings.user

@@ -0,0 +1,13 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AArgumentSpecificationsFactory_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F402b2077f38742cb9b381ab9e79e493229c00_003F4c_003Fc2df050d_003FArgumentSpecificationsFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFixtureMethodRunner_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fd4ca1e5a92fc75932c8ddef25b934f48ed8015686ce1b79256171845d4291_003FFixtureMethodRunner_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F271b8d2d9d94e13a469269f46758864f64ad438c3174f74d4a591af6410e96b_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AILogger_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fcc50fd1d894d4d86ab26ac07ac0af5ec10718_003F51_003F119e8ab7_003FILogger_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ALoggerExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F81cc2099363ff84f1614136d278beb5bed411454404ec1ec16856d72b0d434e9_003FLoggerExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMethodBaseInvoker_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fd6b757e154dd7f8c23e0e785431c97a76e4b9c6bdae38b978238421dbab55d_003FMethodBaseInvoker_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APhysicalFileProvider_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F56b5e122ae4fd723ecae7aec5bf0c71d9181bebd75b4e96887f832e7d81c84_003FPhysicalFileProvider_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARedundantArgumentMatcherException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FLibrary_003FApplication_0020Support_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F402b2077f38742cb9b381ab9e79e493229c00_003F4f_003F17f5c53d_003FRedundantArgumentMatcherException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/Environment/Highlighting/HighlightingSourceSnapshotLocation/@EntryValue">/Users/harelmazor/Library/Caches/JetBrains/Rider2024.3/resharper-host/temp/Rider/vAny/CoverageData/_ElevationWebApi.1119170593/Snapshot/snapshot.utdcvr</s:String>
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=85cfbcaa_002D2803_002D4875_002D8b07_002D402dee8ff687/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" IsActive="True" Name="Test1" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;
+  &lt;Project Location="/Users/harelmazor/dev/ElevationWebAPI/ElevationWebApiTests" Presentation="&amp;lt;ElevationWebApiTests&amp;gt;" /&gt;
+&lt;/SessionState&gt;</s:String></wpf:ResourceDictionary>

ElevationWebApiTests/ElevationWebApiTests.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="MSTest.Sdk/3.6.1">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <LangVersion>latest</LangVersion>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <UseVSTest>true</UseVSTest>
+        <RootNamespace>ElevationWebAPITests</RootNamespace>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\src\ElevationWebApi.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="9.0.6" />
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.6" />
+      <PackageReference Include="NSubstitute" Version="5.3.0" />
+    </ItemGroup>
+
+</Project>

ElevationWebApiTests/InMemoryElevationProviderTests.cs

@@ -0,0 +1,45 @@
+using System.Runtime.CompilerServices;
+using ElevationWebApi;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.FileProviders;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+
+namespace ElevationWebApiTests;
+
+[TestClass]
+public class InMemoryElevationProviderTests
+{
+    private InMemoryElevationProvider provider;
+    
+    private string? GetSourceFileDirectory([CallerFilePath] string sourceFilePath = "")
+    {
+        return Path.GetDirectoryName(sourceFilePath);
+    }
+    
+    [TestInitialize]
+    public void Initialize()
+    {
+        var hosting = Substitute.For<IWebHostEnvironment>();
+        hosting.ContentRootFileProvider = new PhysicalFileProvider( GetSourceFileDirectory()!);
+        var logger = Substitute.For<ILogger<InMemoryElevationProvider>>();
+        provider = new InMemoryElevationProvider(hosting, logger);
+    }
+    
+    
+    [TestMethod]
+    public void CheckElevationAt1_1_ShouldReturnFromFile()
+    {
+        provider.Initialize().Wait();
+        var elevation = provider.GetElevation([[1.0, 1.0]]).Result;
+        Assert.AreEqual(elevation[0], 1291, 0.1);
+    }
+    
+    [TestMethod]
+    public void CheckElevationAt2_2_ShouldBeZero()
+    {
+        provider.Initialize().Wait();
+        var elevation = provider.GetElevation([[2.0, 2.0]]).Result;
+        Assert.AreEqual(elevation[0], 0);
+    }
+}

ElevationWebApiTests/MemoryMapElevationProviderTests.cs

@@ -0,0 +1,46 @@
+using ElevationWebApi;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.FileProviders;
+using NSubstitute;
+using Microsoft.Extensions.Logging;
+using System.Runtime.CompilerServices;
+
+namespace ElevationWebApiTests;
+
+[TestClass]
+public sealed class MemoryMapElevationProviderTests
+{
+    private MemoryMapElevationProvider provider;
+    
+    private string? GetSourceFileDirectory([CallerFilePath] string sourceFilePath = "")
+    {
+        return Path.GetDirectoryName(sourceFilePath);
+    }
+
+    
+    [TestInitialize]
+    public void Initialize()
+    {
+        var hosting = Substitute.For<IWebHostEnvironment>();
+        hosting.ContentRootFileProvider = new PhysicalFileProvider( GetSourceFileDirectory()!);
+        var logger = Substitute.For<ILogger<MemoryMapElevationProvider>>();
+        provider = new MemoryMapElevationProvider(hosting, logger);
+    }
+    
+    
+    [TestMethod]
+    public void CheckElevationAt1_1_ShouldReturnFromFile()
+    {
+        provider.Initialize().Wait();
+        var elevation = provider.GetElevation([[1.0, 1.0]]).Result;
+        Assert.AreEqual(elevation[0], 1291, 0.1);
+    }
+    
+    [TestMethod]
+    public void CheckElevationAt2_2_ShouldBeZero()
+    {
+        provider.Initialize().Wait();
+        var elevation = provider.GetElevation([[2.0, 2.0]]).Result;
+        Assert.AreEqual(elevation[0], 0);
+    }
+}

ElevationWebApiTests/elevation-cache/N01E001.hgt

@@ -1,6 +1,6 @@
 # ElevationWebApi
 
-A .net core elevation service that uses memory mapped hgt files to return elevation information for a given latitude and longitude array.
+A .net core elevation service that uses memory mapped hgt files or in memory files to return elevation information for a given latitude and longitude array.
 
 To build the docker image: 
 ```
@@ -10,12 +10,12 @@ docker run -p 12345:8080 -v /hgt-files-folder:/app/elevation-cache elevationweba
 
 Now surf to `localhost:12345/swagger/` to get a simple UI to interact with the elevation service
 
-This repository was only tested for elevation for Israel, which is a small country and the implementation be might be enough, it might not be the case for bigger countries.
+This service was tested on the entire world with memory mapped files and on a small country with in memory files.
 
 This service supports both GET and POST methods for getting the elevation.
 You can place in the elevation folder zip or bz2 compressed files and the service will decompress them when booting up.
 
-This docker file is also available on docker hub: `israelhikingmap/elevationwebapi`
+This docker image is also available on docker hub: `israelhikingmap/elevationwebapi`
 
 You can choose between two types of elevation providers by specifying the `ELEVATION_PROVIDER` environment variable:
 `MMAP` - For memory mapped elevation provider

README.md

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.IO;
 using System.IO.MemoryMappedFiles;
 using System.Linq;
@@ -20,7 +21,8 @@ public class MemoryMapElevationProvider : IElevationProvider
     {
         private readonly ILogger<MemoryMapElevationProvider> _logger;
         private readonly IFileProvider _fileProvider;
-        private readonly ConcurrentDictionary<Coordinate, Task<FileAndSamples>> _initializationTaskPerLatLng;
+        private readonly ConcurrentDictionary<Coordinate, Task<FileAndSamples>> _mappedFilesCache;
+        private readonly Dictionary<Coordinate, (string path, long length)> _initializationAvailableFiles;
 
         /// <summary>
         /// Constructor
@@ -31,23 +33,24 @@ public MemoryMapElevationProvider(IWebHostEnvironment webHostEnvironment, ILogge
         {
             _logger = logger;
             _fileProvider = webHostEnvironment.ContentRootFileProvider;
-            _initializationTaskPerLatLng = new();
+            _mappedFilesCache = new();
+            _initializationAvailableFiles = new();
         }
 
         /// <summary>
         /// Initializes the provider by reading the elevation-cache directory,
         /// extracting the zip/bz2 files if needed and memory mapping them to a dictionary
         /// </summary>
-        public async Task Initialize()
+        public Task Initialize()
         {
             if (!ElevationHelper.ValidateFolder(_fileProvider, _logger))
             {
-                return;
+                return Task.CompletedTask;
             }
 
             ElevationHelper.UnzipIfNeeded(_fileProvider, _logger);
             var hgtFiles = _fileProvider.GetDirectoryContents(ElevationHelper.ELEVATION_CACHE)
-                .Where(f => f.PhysicalPath.EndsWith(".hgt")).ToArray();
+                .Where(f => f.PhysicalPath != null && f.PhysicalPath.EndsWith(".hgt")).ToArray();
             _logger.LogInformation($"Found {hgtFiles.Length} hgt files.");
             foreach (var hgtFile in hgtFiles)
             {
@@ -57,11 +60,12 @@ public async Task Initialize()
                     _logger.LogWarning($"Ignoring file: {hgtFile.Name}");
                     continue;
                 }
-                _initializationTaskPerLatLng[key] = Task.Run(() => new FileAndSamples(MemoryMappedFile.CreateFromFile(hgtFile.PhysicalPath, FileMode.Open), ElevationHelper.SamplesFromLength(hgtFile.Length)));
-            }
 
-            await Task.WhenAll(_initializationTaskPerLatLng.Values);
+                _initializationAvailableFiles[key] = (hgtFile.PhysicalPath, hgtFile.Length); 
+            }
+            
             _logger.LogInformation("Initialization complete.");
+            return Task.CompletedTask;
         }
 
         /// <summary>
@@ -74,12 +78,17 @@ public Task<double[]> GetElevation(double[][] latLngs)
             var tasks = latLngs.Select(async latLng =>
             {
                 var key = new Coordinate(Math.Floor(latLng[0]), Math.Floor(latLng[1]));
-                if (_initializationTaskPerLatLng.ContainsKey(key) == false)
+                if (_initializationAvailableFiles.TryGetValue(key, out (string path, long length) pathAndLength) == false)
                 {
                     return 0;
                 }
 
-                var info = await _initializationTaskPerLatLng[key];
+                if (!_mappedFilesCache.ContainsKey(key))
+                {
+                    _logger.LogInformation($"Loading {pathAndLength.path} into memory mapped cache");
+                    _mappedFilesCache[key] = Task.Run(() => new FileAndSamples(MemoryMappedFile.CreateFromFile(pathAndLength.path, FileMode.Open), ElevationHelper.SamplesFromLength(pathAndLength.length)));
+                }
+                var info = await _mappedFilesCache[key];
 
                 var exactLocation = new Coordinate(Math.Abs(latLng[0] - key.X) * (info.Samples - 1),
                     (1 - Math.Abs(latLng[1] - key.Y)) * (info.Samples - 1));
@@ -100,8 +109,8 @@ public Task<double[]> GetElevation(double[][] latLngs)
         /// Get the elevation of the two adjacent indices (i,j), (i, j+1)
         /// Then converts them to 3d points 
         /// </summary>
-        /// <param name="i">i Index in file</param>
-        /// <param name="j">j index in file</param>
+        /// <param name="i">I Index in file</param>
+        /// <param name="j">J index in file</param>
         /// <param name="info">The info relevant to the file</param>
         /// <returns></returns>
         private (CoordinateZ, CoordinateZ) GetElevationForLocation(int i, int j, FileAndSamples info)
@@ -111,7 +120,7 @@ public Task<double[]> GetElevation(double[][] latLngs)
                 var byteIndex = (i1 * info.Samples + j1) * 2;
                 using var stream = info.File.CreateViewStream(byteIndex, 4);
                 byte[] byteArray = new byte[4];
-                stream.Read(byteArray);
+                stream.ReadExactly(byteArray);
                 return byteArray;
             });
         }