view pkg/geoserver/boot.go @ 3678:8f58851927c0

client: make layer factory only return new layer config for individual maps instead of each time it is invoked. The purpose of the factory was to support multiple maps with individual layers. But returning a new config each time it is invoked leads to bugs that rely on the layer's state. Now this factory reuses the same objects it created before, per map.
author Markus Kottlaender <markus@intevation.de>
date Mon, 17 Jun 2019 17:31:35 +0200
parents bf5ab7a069e2
children 36129677ff24
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2018, 2019 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package geoserver

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html/charset"

	"gemma.intevation.de/gemma/pkg/config"
	"gemma.intevation.de/gemma/pkg/models"
)

const (
	workspaceName         = "gemma"
	datastoreName         = "gemma"
	databaseScheme        = "waterway"
	databaseType          = "postgis"
	primaryKeyMetadataTbl = "waterway.gt_pk_metadata"
)

const (
	startupSQL = `SELECT public.setrole('${user,'||encode('waterway_user', 'hex')||'}')`
	closeupSQL = `RESET ROLE`
)

func basicAuth(user, password string) func(req *http.Request) {
	return func(req *http.Request) {
		req.SetBasicAuth(user, password)
	}
}

func toStream(x interface{}) io.Reader {
	var buf bytes.Buffer

	if err := json.NewEncoder(&buf).Encode(x); err != nil {
		// Should not happen
		log.Printf("warn: bad JSON: %v\n", err)
	}
	return bytes.NewReader(buf.Bytes())
}

func asJSON(req *http.Request) {
	req.Header.Set("Content-Type", "application/json")
}

func asContentType(req *http.Request, contentType string) {
	req.Header.Set("Content-Type", contentType)
}

func ensureWorkspace() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	// Probe  workspace.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+".json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusNotFound {
		log.Println("info: workspace " + workspaceName + " already exists.")
		return nil
	}

	// Create workspace

	log.Println("info: creating workspace " + workspaceName)

	const createJSON = `{"workspace":{"name":"` + workspaceName + `"}}`

	req, err = http.NewRequest(
		http.MethodPost,
		geoURL+"/rest/workspaces",
		strings.NewReader(createJSON))
	if err != nil {
		return err
	}
	auth(req)
	asJSON(req)
	if resp, err = http.DefaultClient.Do(req); err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		err = fmt.Errorf("Status code '%s' (%d)",
			http.StatusText(resp.StatusCode),
			resp.StatusCode)
	}

	return err
}

func ensureDataStore() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	// Probe datastore.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+
			"/datastores/"+datastoreName+".json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusNotFound {
		log.Println("info: datastore " + datastoreName + " already exists.")
		return nil
	}

	// Create datastore.
	log.Println("info: creating datastore " + datastoreName)

	type entry struct {
		Key   interface{} `json:"@key"`
		Value interface{} `json:"$"`
	}

	// Create datastore.
	ds := map[string]interface{}{
		"dataStore": map[string]interface{}{
			"name": datastoreName,
			"connectionParameters": map[string]interface{}{
				"entry": []entry{
					{"host", config.DBHost()},
					{"port", config.DBPort()},
					{"database", config.DBName()},
					{"schema", databaseScheme},
					{"user", config.DBUser()},
					{"passwd", config.DBPassword()},
					{"dbtype", databaseType},
					{"Primary key metadata table", primaryKeyMetadataTbl},
					{"Session startup SQL", startupSQL},
					{"Session close-up SQL", closeupSQL},
					{"validate connections", true},
					{"Estimated extends", false},
				},
			},
		},
	}

	req, err = http.NewRequest(
		http.MethodPost,
		geoURL+"/rest/workspaces/"+workspaceName+"/datastores",
		toStream(ds))
	if err != nil {
		return err
	}
	auth(req)
	asJSON(req)
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		err = fmt.Errorf("Status code '%s' (%d)",
			http.StatusText(resp.StatusCode),
			resp.StatusCode)
	}

	return err
}

func ensureFeatures() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	tables := models.InternalServices.Filter(models.IntWFS)
	if len(tables) == 0 {
		log.Println("info: no tables to publish")
		return nil
	}

	log.Printf("info: number of tables to publish %d\n", len(tables))

	var features struct {
		FeatureTypes struct {
			FeatureType []struct {
				Name string `json:"name"`
			} `json:"featureType"`
		} `json:"featureTypes"`
	}

	hasFeature := func(name string) bool {
		for _, ft := range features.FeatureTypes.FeatureType {
			if ft.Name == name {
				return true
			}
		}
		return false
	}

	// Fetch all featuretypes.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+
			"/datastores/"+datastoreName+
			"/featuretypes.json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	err = json.NewDecoder(resp.Body).Decode(&features)
	resp.Body.Close()
	if err != nil {
		// XXX: Quirk in the JSON return by GeoServer:
		// If there are no features in the datastore
		// featureType deserializes to an empty string ""
		// instead of an empty array *palmface*.
		// So assume there no features.
		hasFeature = func(string) bool { return false }
	}

	var already []string

	defer func() {
		if len(already) > 0 {
			log.Printf("info: already having featuretypes: %s\n",
				strings.Join(already, ", "))
		}
	}()

	for i := range tables {
		table := tables[i].Name

		if hasFeature(table) {
			already = append(already, table)
			continue
		}

		// Create featuretype.
		log.Printf("info: creating featuretype %s.\n", table)

		// Create featuretype
		ft := map[string]interface{}{
			"featureType": map[string]interface{}{
				"name":       table,
				"nativeName": table,
				"title":      table,
			},
		}

		req, err := http.NewRequest(
			http.MethodPost,
			geoURL+"/rest/workspaces/"+workspaceName+
				"/datastores/"+datastoreName+
				"/featuretypes",
			toStream(ft))
		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("Status code '%s' (%d)",
				http.StatusText(resp.StatusCode),
				resp.StatusCode)
		}
	}

	return nil
}

func deleteWorkspace() error {

	// Should we delete our workspace first?
	if !config.GeoServerClean() {
		return nil
	}

	log.Println("info: delete workspace " + workspaceName)
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	req, err := http.NewRequest(
		http.MethodDelete,
		geoURL+"/rest/workspaces/"+workspaceName+"?recurse=true",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	_, err = http.DefaultClient.Do(req)
	return err
}

type styles struct {
	Styles struct {
		Style []struct {
			Name string `json:"name"`
		} `json:"style"`
	} `json:"styles"`
}

func (s *styles) hasStyle(name string) bool {
	for i := range s.Styles.Style {
		if s.Styles.Style[i].Name == name {
			return true
		}
	}
	return false
}

func (s *styles) load() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+"/styles.json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Fetch all styles
	// XXX: Avoid error checking due to quirks with featuretypes.
	json.NewDecoder(resp.Body).Decode(s)
	return nil
}

func updateStyle(entry *models.IntEntry, create bool) error {

	log.Printf("info: creating style %s\n", entry.Name)

	// Try to load the style data.
	data, err := entry.LoadStyle()
	if err != nil {
		return err
	}

	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	styleURL := geoURL + "/rest/workspaces/" + workspaceName +
		"/styles"

	// First create style

	type Style struct {
		Name     string `json:"name"`
		Filename string `json:"filename"`
	}

	var styleFilename = struct {
		Style Style `json:"style"`
	}{
		Style: Style{
			Name:     entry.Name,
			Filename: entry.Name + ".sld",
		},
	}

	if create {
		req, err := http.NewRequest(
			http.MethodPost,
			styleURL,
			toStream(&styleFilename))

		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("Unable to create style %s (%s)",
				entry.Name,
				http.StatusText(resp.StatusCode))
		}
	}

	// Second upload data

	req, err := http.NewRequest(
		http.MethodPut,
		styleURL+"/"+url.PathEscape(entry.Name),
		strings.NewReader(data))
	if err != nil {
		return err
	}
	auth(req)
	if isSymbologyEncoding(data) {
		asContentType(req, "application/vnd.ogc.se+xml")
	} else {
		asContentType(req, "application/vnd.ogc.sld+xml")
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("cannot upload style %s (%s)",
			entry.Name, http.StatusText(resp.StatusCode))
	}

	// Third associate with layer

	if create {
		req, err := http.NewRequest(
			http.MethodPost,
			geoURL+"/rest/layers/"+
				url.PathEscape(workspaceName+":"+entry.Name)+
				"/styles?default=true",
			toStream(&styleFilename))
		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)

		resp, err = http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("cannot connect style %s with layer (%s)",
				entry.Name, http.StatusText(resp.StatusCode))
		}
	}

	return nil
}

// isSymbologyEncoding tries to figure out if its plain SLD or SE.
func isSymbologyEncoding(data string) bool {
	decoder := xml.NewDecoder(strings.NewReader(data))
	decoder.CharsetReader = charset.NewReaderLabel

	for {
		tok, err := decoder.Token()
		switch {
		case tok == nil && err == io.EOF:
			return false
		case err != nil:
			log.Printf("warn: invalid XML: %v\n", err)
			return false
		}
		if t, ok := tok.(xml.StartElement); ok &&
			t.Name.Space == "http://www.opengis.net/se" {
			return true
		}
	}
}

func ensureStyles() error {
	log.Println("info: creating styles")

	var stls styles
	if err := stls.load(); err != nil {
		return err
	}

	entries := models.InternalServices.Filter(
		models.IntAnd(
			models.IntWMS,
			models.IntWithStyle))

	var already []string

	defer func() {
		if len(already) > 0 {
			log.Printf("info: already having styles: %s\n",
				strings.Join(already, ", "))
		}
	}()

	for i := range entries {
		entry := &entries[i]
		if stls.hasStyle(entry.Name) {
			already = append(already, entry.Name)
			continue
		}
		if err := updateStyle(entry, true); err != nil {
			return err
		}
	}

	return nil
}

// PrepareGeoServer sets up the GeoServer to work together with the gemma server.
// It sets up a workspace, a datastore and exposes the features and styles.
func PrepareGeoServer() error {

	if config.DBUser() == "" {
		log.Println("info: Need metamorphic db user to configure GeoServer")
		return nil
	}

	if config.GeoServerURL() == "" {
		log.Println("info: No URL to GeoServer configured")
		return nil
	}

	for _, ensure := range []func() error{
		deleteWorkspace,
		ensureWorkspace,
		ensureDataStore,
		ensureFeatures,
		ensureStyles,
	} {
		if err := ensure(); err != nil {
			return err
		}
	}

	return nil
}

// ReconfigureStyle returns a function to update a style
// in the GeoServer to be in sync with the database.
func ReconfigureStyle(name string) {
	Reconfigure(func() error {
		var stls styles
		if err := stls.load(); err != nil {
			return err
		}

		entries := models.InternalServices.Filter(
			models.IntAnd(
				models.IntWMS,
				models.IntWithStyle,
				models.IntByName(name)))

		for i := range entries {
			entry := &entries[i]
			create := !stls.hasStyle(entry.Name)
			if err := updateStyle(entry, create); err != nil {
				return err
			}
		}

		return nil
	})
}