tiledConusProcessing/main.R at main · ncss-tech/tiledConusProcessing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
##
## Draft workflow for creating thematic maps from CONUS gNATSGO mukey grid + SDA
##

## TODO:
# * use fSSURGO instead of gSTATSGO
# * profile CPU, RAM, disk, network resources
# * finish categorical variable handling
# * better selection of aggregation type
# * add depth interval to file name


## Notes:
# * be sure to disable VPN and free as much RAM as possible
#
# government machine, at the office
#  * 26 minutes for a single component level property
#  * 37 minutes for 2 hz level properties
#  * 45 minutes for 3 hz level properties
#
#  * 25 minutes for mosaic/re-sample of single property
#  * 30 minutes for 3 hz level properties
#

# 2024-03-26:
#
# gNATSGO 30m:
#  * 68 seconds / tile (6 hz level properties)
#  * 1 hour for 6 hz level properties (SDA / slow internet)

# STATSGO 300m:
# * 74 seconds for 6 hz level properties (SDA / slow internet)
# *

# gSSURGO 30m (SoilWeb 4-1)
#  * 43 minutes for 6 hz level properties (SDA)
#

# 2025-12-17
#
#


## create tile systems
## these are already in place
# source('tiling-grid-A.R')

## tile CONUS mukey grid
# only need to do this once per FY snapshot
# synced with other operations via config.R
#
# gSSURGO/gNATSGO 30m : ~ 10 minutes
# STATSGO 300m : ~ 2 minutes
system.time(
  source('tile-mukey-grid.R')
)


## collect unique map unit keys
# only need to do this once per FY / source grid
#
# GFE fSSURGO 30m: 2.7 minutes
source('prepare-unique-mukey.R')


## prepare LUT for unique mukey and all variables
# do this each time config.R changes
#
# GFE fSSURGO 30m: 95 seconds (parallel) | 17 minutes (serial)
source('prepare-LUT.R')


## process tiles, create thematic grid tiles
# variables stored in config.R
#
# 2026-02-13
# terra terra_1.8-97
# GFE fSSURGO 30m: 13 minutes !!!  (6 properties)
system.time(
  source('process-tiles.R')
)

## mosaic / re-sample tiles

# 2025-12-19
# fSSURGO 30m:        21 minutes (6 properties)

# 2026-02-13
# fSSURGO 30m:        19.5 minutes (6 properties)


# gSSURGO 30m:        10 minutes (1 property)
# gNATSGO 30m:        36 minutes (6 properties)
# STATSGO 300m:       32 seconds (6 properties)
#
# gSSURGO (SoilWeb):  10 minutes (6 properties)
#
# variables stored in config.R
system.time(
  source('mosaic-tiles.R')
)