Merge branch 'v3.1.0'

Author: lambdajack

.gitignore

@@ -1,17 +1,5 @@
-mbtiles
-openmaptiles
-tippecanoe
-tilemaker
-pbf
-coastline
-water-polygons*
 test.json
-my.json
-tmp
 data
 main
-REPORT.txt
-.vscode
-
-!/configs/tilemaker
-!third_party/*
+bin
+.vscode

.gitmodules

@@ -0,0 +1,10 @@
+## All contributions are welcome.
+
+#### We could particularly use help with:
+
+1. macOS testing / improvements.
+
+### Dev:
+
+- Build using 'make' and remember to set the $(VERSION) variable appropriately.
+- Testing may have to be done on built binaries if your golang and docker permissions clash.

CONTRIBUTING.md

@@ -0,0 +1,17 @@
+VERSION := v3.1.0
+LDF := "-X github.com/lambdajack/sequentially-generate-planet-mbtiles/cmd/sequentially-generate-planet-mbtiles.sgpmVersion=$(VERSION)"
+
+all: clean darwin linux windows
+
+darwin:
+	GOOS=darwin GOARCH=amd64 go build -o bin/sequentially-generate-planet-mbtiles--darwin-amd64-$(VERSION) -ldflags $(LDF)
+
+linux:
+	GOOS=linux GOARCH=amd64 go build -o bin/sequentially-generate-planet-mbtiles--unix-amd64-$(VERSION) -ldflags $(LDF)
+	GOOS=linux GOARCH=arm go build -o bin/sequentially-generate-planet-mbtiles--unix-arm64-$(VERSION) -ldflags $(LDF)
+
+windows:
+	GOOS=windows GOARCH=amd64 go build -o bin/sequentially-generate-planet-mbtiles--win-amd64-$(VERSION).exe -ldflags $(LDF)
+
+clean:
+	rm -rf bin/

Makefile

@@ -1,5 +1,4 @@
-FROM ubuntu:20.04
-ARG DEBIAN_FRONTEND=noninteractive
+FROM debian:bookworm-slim as builder
 
 RUN apt-get update
 RUN apt-get update && apt-get install -y \

README.md

@@ -0,0 +1,164 @@
+package sequentiallygenerateplanetmbtiles
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/lambdajack/lj_go/pkg/lj_json"
+)
+
+type configuration struct {
+	srcFileProvided  bool
+	PbfFile          string `json:"pbfFile"`
+	WorkingDir       string `json:"workingDir"`
+	OutDir           string `json:"outDir"`
+	ExcludeOcean     bool   `json:"excludeOcean"`
+	ExcludeLanduse   bool   `json:"excludeLanduse"`
+	TilemakerConfig  string `json:"TilemakerConfig"`
+	TilemakerProcess string `json:"TilemakerProcess"`
+	MaxRamMb         uint64 `json:"maxRamMb"`
+	OutAsDir         bool   `json:"outAsDir"`
+	SkipSlicing      bool   `json:"skipSlicing"`
+	MergeOnly        bool   `json:"mergeOnly"`
+	SkipDownload     bool   `json:"skipDownload"`
+}
+
+func initConfig() {
+	if fl.config == "" {
+		setConfigByFlags()
+	} else {
+		setConfigByJSON()
+	}
+
+	verifyPaths()
+
+	cfg.PbfFile = convertAbs(cfg.PbfFile)
+	cfg.WorkingDir = convertAbs(cfg.WorkingDir)
+	cfg.OutDir = convertAbs(cfg.OutDir)
+}
+
+func setConfigByJSON() {
+	err := lj_json.DecodeTo(cfg, fl.config, 1000)
+	if err != nil {
+		log.Printf("Unable to decode config: %s", err)
+		os.Exit(exitInvalidJSON)
+	}
+	if cfg.MaxRamMb == 0 {
+		cfg.MaxRamMb = getRam()
+	}
+}
+
+func setConfigByFlags() {
+	cfg.PbfFile = fl.pbfFile
+	cfg.WorkingDir = fl.workingDir
+	cfg.OutDir = fl.outDir
+	cfg.ExcludeOcean = fl.excludeOcean
+	cfg.ExcludeLanduse = fl.excludeLanduse
+	cfg.TilemakerConfig = fl.tilemakerConfig
+	cfg.TilemakerProcess = fl.tilemakerProcess
+	cfg.MaxRamMb = getRam()
+	cfg.OutAsDir = fl.outAsDir
+	cfg.SkipSlicing = fl.skipSlicing
+	cfg.MergeOnly = fl.mergeOnly
+	cfg.SkipDownload = fl.skipDownload
+}
+
+func setTmPaths() {
+	if cfg.TilemakerProcess == "" || strings.ToLower(cfg.TilemakerProcess) == "tileserver-gl-basic" {
+		cfg.TilemakerProcess = filepath.Join(pth.temp, "tilemaker", "resources", "process-openmaptiles.lua")
+		log.Println("using tileserver-gl-basic style target")
+	}
+
+	if cfg.TilemakerProcess == "sgpm-bright" {
+		cfg.TilemakerProcess = filepath.Join(pth.temp, "tilemaker", "resources", "process-bright.lua")
+		log.Println("using sgpm-bright style target")
+	}
+
+	if cfg.TilemakerConfig == "" {
+		cfg.TilemakerConfig = filepath.Join(pth.temp, "tilemaker", "resources", "config-openmaptiles.json")
+	}
+
+	cfg.TilemakerConfig = convertAbs(cfg.TilemakerConfig)
+	cfg.TilemakerProcess = convertAbs(cfg.TilemakerProcess)
+}
+
+func verifyPaths() {
+	if cfg.PbfFile != "" {
+		if _, err := os.Stat(cfg.PbfFile); os.IsNotExist(err) {
+			log.Fatalf("planet file does not exist: %s", cfg.PbfFile)
+		}
+		cfg.srcFileProvided = true
+	} else {
+		if fl.test {
+			cfg.PbfFile = filepath.Join(cfg.WorkingDir, "pbf", "morocco-latest.osm.pbf")
+		} else {
+			cfg.PbfFile = filepath.Join(cfg.WorkingDir, "pbf", "planet-latest.osm.pbf")
+		}
+	}
+
+	if cfg.TilemakerConfig != "" {
+		if _, err := os.Stat(cfg.TilemakerConfig); os.IsNotExist(err) {
+			log.Fatalf("tilemaker config does not exist: %s", cfg.TilemakerConfig)
+		}
+	}
+
+	if cfg.TilemakerProcess != "" && cfg.TilemakerProcess != "sgpm-bright" && cfg.TilemakerProcess != "tileserver-gl-basic" {
+		if _, err := os.Stat(cfg.TilemakerProcess); os.IsNotExist(err) {
+			log.Fatalf("tilemaker process does not exist: %s", cfg.TilemakerProcess)
+		}
+	}
+}
+
+func getRam() uint64 {
+	if fl.maxRamMb != 0 {
+		return fl.maxRamMb
+	}
+
+	var memTotalKb uint64
+
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		log.Printf("Unable to open /proc/meminfo: %s", err)
+		memTotalKb = 4096 * 1024
+	} else {
+		defer f.Close()
+
+		scanner := bufio.NewScanner(f)
+
+		memTotalStr := ""
+
+		for scanner.Scan() {
+			if strings.HasPrefix(scanner.Text(), "MemTotal:") {
+				memTotalStr = regexp.MustCompile(`[0-9]+`).FindString(scanner.Text())
+			}
+		}
+
+		memTotalKb, err = strconv.ParseUint(memTotalStr, 10, 64)
+		if err != nil {
+			log.Printf("Unable to parse MemTotal: %s", err)
+			memTotalKb = 4096 * 1024
+		}
+	}
+
+	if (memTotalKb / 1024) < 2048 {
+		log.Printf("system ram is less than 2 GB; this is dangerously low, but we will do our best!")
+		return 1024
+	}
+
+	memTotalMb := memTotalKb / 1024
+
+	totalRamToUse := memTotalMb - 2048
+
+	if totalRamToUse < 1024 {
+		totalRamToUse = 1024
+	}
+
+	log.Printf("attempting to use not more than %d MB of ram", totalRamToUse)
+
+	return totalRamToUse
+}

build/osmium/Dockerfile

@@ -0,0 +1,91 @@
+package sequentiallygenerateplanetmbtiles
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+func init() {
+
+	tmpDir := filepath.Join(os.TempDir(), "sequentially-generate-planet-mbtiles")
+
+	err := os.MkdirAll(tmpDir, os.ModePerm)
+	if err != nil {
+		panic("Could not make os.TempDir/sequentially-generate-planet-mbtiles. Permissions issue? TempleOS? Expect many tests to fail.")
+	}
+
+	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
+		panic("Could not find os.TempDir. Permissions issue? TempleOS? Expect many tests to fail.")
+	}
+
+	cfg = &configuration{
+		PbfFile:          "", // CHANGE TO TEST FILE ONCE SET U
+		WorkingDir:       tmpDir,
+		OutDir:           filepath.Join(tmpDir, "out"),
+		ExcludeOcean:     true,
+		ExcludeLanduse:   true,
+		TilemakerConfig:  "",
+		TilemakerProcess: "",
+		MaxRamMb:         1,
+	}
+
+	fmt.Printf("cfg initialised: %+v\n", cfg)
+}
+
+func TestSetInvalidConfigByJSON(t *testing.T) {
+	// Test run in separate processes so as not to pollute other tests
+
+	invalidConfig := "../../configs/test/TEST_INVALID.json"
+
+	if _, err := os.Stat(invalidConfig); os.IsNotExist(err) {
+		t.Errorf("TestSetConfigByJSON test config file does not exist")
+	}
+
+	if os.Getenv("TEST_SET_CONFIG_BY_JSON") == "1" {
+		os.Args = append(os.Args, "-c", invalidConfig)
+		flag.Parse()
+		setConfigByJSON()
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run=TestSetInvalidConfigByJSON")
+	cmd.Env = append(os.Environ(), "TEST_SET_CONFIG_BY_JSON=1")
+	defer os.Unsetenv("TEST_SET_CONFIG_BY_JSON")
+	err := cmd.Run()
+
+	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
+		if e.ExitCode() == exitInvalidJSON {
+			return
+		}
+	}
+
+	t.Fatalf("process ran with err %v, want exit status %v", err, exitInvalidJSON)
+}
+
+func TestSetValidConfigByJSON(t *testing.T) {
+	validConfig := "../../configs/test/TEST_VALID.json"
+
+	if _, err := os.Stat(validConfig); os.IsNotExist(err) {
+		t.Errorf("TestSetConfigByJSON test config file does not exist")
+	}
+
+	if os.Getenv("TEST_SET_VALID_CONFIG_BY_JSON") == "1" {
+		t.Log(os.Args)
+		os.Args = append(os.Args, "-c", validConfig)
+		flag.Parse()
+		setConfigByJSON()
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run=TestSetValidConfigByJSON")
+	cmd.Env = append(os.Environ(), "TEST_SET_VALID_CONFIG_BY_JSON=1")
+	defer os.Unsetenv("TEST_SET_VALID_CONFIG_BY_JSON")
+	err := cmd.Run()
+	if err != nil {
+		t.Fatalf("process ran with err %v, want %v", err, nil)
+	}
+}