view pkg/geoserver/boot.go @ 3306:bf5ab7a069e2

More compact logging of GeoServer booting.
author Sascha L. Teichmann <sascha.teichmann@intevation.de>
date Fri, 17 May 2019 11:24:06 +0200
parents fd14f149696c
children 36129677ff24
line wrap: on
line source

// This is Free Software under GNU Affero General Public License v >= 3.0
// without warranty, see README.md and license for details.
//
// SPDX-License-Identifier: AGPL-3.0-or-later
// License-Filename: LICENSES/AGPL-3.0.txt
//
// Copyright (C) 2018, 2019 by via donau
//   – Österreichische Wasserstraßen-Gesellschaft mbH
// Software engineering by Intevation GmbH
//
// Author(s):
//  * Sascha L. Teichmann <sascha.teichmann@intevation.de>

package geoserver

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"golang.org/x/net/html/charset"

	"gemma.intevation.de/gemma/pkg/config"
	"gemma.intevation.de/gemma/pkg/models"
)

const (
	workspaceName         = "gemma"
	datastoreName         = "gemma"
	databaseScheme        = "waterway"
	databaseType          = "postgis"
	primaryKeyMetadataTbl = "waterway.gt_pk_metadata"
)

const (
	startupSQL = `SELECT public.setrole('${user,'||encode('waterway_user', 'hex')||'}')`
	closeupSQL = `RESET ROLE`
)

func basicAuth(user, password string) func(req *http.Request) {
	return func(req *http.Request) {
		req.SetBasicAuth(user, password)
	}
}

func toStream(x interface{}) io.Reader {
	var buf bytes.Buffer

	if err := json.NewEncoder(&buf).Encode(x); err != nil {
		// Should not happen
		log.Printf("warn: bad JSON: %v\n", err)
	}
	return bytes.NewReader(buf.Bytes())
}

func asJSON(req *http.Request) {
	req.Header.Set("Content-Type", "application/json")
}

func asContentType(req *http.Request, contentType string) {
	req.Header.Set("Content-Type", contentType)
}

func ensureWorkspace() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	// Probe  workspace.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+".json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusNotFound {
		log.Println("info: workspace " + workspaceName + " already exists.")
		return nil
	}

	// Create workspace

	log.Println("info: creating workspace " + workspaceName)

	const createJSON = `{"workspace":{"name":"` + workspaceName + `"}}`

	req, err = http.NewRequest(
		http.MethodPost,
		geoURL+"/rest/workspaces",
		strings.NewReader(createJSON))
	if err != nil {
		return err
	}
	auth(req)
	asJSON(req)
	if resp, err = http.DefaultClient.Do(req); err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		err = fmt.Errorf("Status code '%s' (%d)",
			http.StatusText(resp.StatusCode),
			resp.StatusCode)
	}

	return err
}

func ensureDataStore() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	// Probe datastore.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+
			"/datastores/"+datastoreName+".json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusNotFound {
		log.Println("info: datastore " + datastoreName + " already exists.")
		return nil
	}

	// Create datastore.
	log.Println("info: creating datastore " + datastoreName)

	type entry struct {
		Key   interface{} `json:"@key"`
		Value interface{} `json:"$"`
	}

	// Create datastore.
	ds := map[string]interface{}{
		"dataStore": map[string]interface{}{
			"name": datastoreName,
			"connectionParameters": map[string]interface{}{
				"entry": []entry{
					{"host", config.DBHost()},
					{"port", config.DBPort()},
					{"database", config.DBName()},
					{"schema", databaseScheme},
					{"user", config.DBUser()},
					{"passwd", config.DBPassword()},
					{"dbtype", databaseType},
					{"Primary key metadata table", primaryKeyMetadataTbl},
					{"Session startup SQL", startupSQL},
					{"Session close-up SQL", closeupSQL},
					{"validate connections", true},
					{"Estimated extends", false},
				},
			},
		},
	}

	req, err = http.NewRequest(
		http.MethodPost,
		geoURL+"/rest/workspaces/"+workspaceName+"/datastores",
		toStream(ds))
	if err != nil {
		return err
	}
	auth(req)
	asJSON(req)
	resp, err = http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		err = fmt.Errorf("Status code '%s' (%d)",
			http.StatusText(resp.StatusCode),
			resp.StatusCode)
	}

	return err
}

func ensureFeatures() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	tables := models.InternalServices.Filter(models.IntWFS)
	if len(tables) == 0 {
		log.Println("info: no tables to publish")
		return nil
	}

	log.Printf("info: number of tables to publish %d\n", len(tables))

	var features struct {
		FeatureTypes struct {
			FeatureType []struct {
				Name string `json:"name"`
			} `json:"featureType"`
		} `json:"featureTypes"`
	}

	hasFeature := func(name string) bool {
		for _, ft := range features.FeatureTypes.FeatureType {
			if ft.Name == name {
				return true
			}
		}
		return false
	}

	// Fetch all featuretypes.
	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+
			"/datastores/"+datastoreName+
			"/featuretypes.json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	err = json.NewDecoder(resp.Body).Decode(&features)
	resp.Body.Close()
	if err != nil {
		// XXX: Quirk in the JSON return by GeoServer:
		// If there are no features in the datastore
		// featureType deserializes to an empty string ""
		// instead of an empty array *palmface*.
		// So assume there no features.
		hasFeature = func(string) bool { return false }
	}

	var already []string

	defer func() {
		if len(already) > 0 {
			log.Printf("info: already having featuretypes: %s\n",
				strings.Join(already, ", "))
		}
	}()

	for i := range tables {
		table := tables[i].Name

		if hasFeature(table) {
			already = append(already, table)
			continue
		}

		// Create featuretype.
		log.Printf("info: creating featuretype %s.\n", table)

		// Create featuretype
		ft := map[string]interface{}{
			"featureType": map[string]interface{}{
				"name":       table,
				"nativeName": table,
				"title":      table,
			},
		}

		req, err := http.NewRequest(
			http.MethodPost,
			geoURL+"/rest/workspaces/"+workspaceName+
				"/datastores/"+datastoreName+
				"/featuretypes",
			toStream(ft))
		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("Status code '%s' (%d)",
				http.StatusText(resp.StatusCode),
				resp.StatusCode)
		}
	}

	return nil
}

func deleteWorkspace() error {

	// Should we delete our workspace first?
	if !config.GeoServerClean() {
		return nil
	}

	log.Println("info: delete workspace " + workspaceName)
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	req, err := http.NewRequest(
		http.MethodDelete,
		geoURL+"/rest/workspaces/"+workspaceName+"?recurse=true",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	_, err = http.DefaultClient.Do(req)
	return err
}

type styles struct {
	Styles struct {
		Style []struct {
			Name string `json:"name"`
		} `json:"style"`
	} `json:"styles"`
}

func (s *styles) hasStyle(name string) bool {
	for i := range s.Styles.Style {
		if s.Styles.Style[i].Name == name {
			return true
		}
	}
	return false
}

func (s *styles) load() error {
	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	req, err := http.NewRequest(
		http.MethodGet,
		geoURL+"/rest/workspaces/"+workspaceName+"/styles.json",
		nil)
	if err != nil {
		return err
	}
	auth(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Fetch all styles
	// XXX: Avoid error checking due to quirks with featuretypes.
	json.NewDecoder(resp.Body).Decode(s)
	return nil
}

func updateStyle(entry *models.IntEntry, create bool) error {

	log.Printf("info: creating style %s\n", entry.Name)

	// Try to load the style data.
	data, err := entry.LoadStyle()
	if err != nil {
		return err
	}

	var (
		geoURL   = config.GeoServerURL()
		user     = config.GeoServerUser()
		password = config.GeoServerPassword()
		auth     = basicAuth(user, password)
	)

	styleURL := geoURL + "/rest/workspaces/" + workspaceName +
		"/styles"

	// First create style

	type Style struct {
		Name     string `json:"name"`
		Filename string `json:"filename"`
	}

	var styleFilename = struct {
		Style Style `json:"style"`
	}{
		Style: Style{
			Name:     entry.Name,
			Filename: entry.Name + ".sld",
		},
	}

	if create {
		req, err := http.NewRequest(
			http.MethodPost,
			styleURL,
			toStream(&styleFilename))

		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("Unable to create style %s (%s)",
				entry.Name,
				http.StatusText(resp.StatusCode))
		}
	}

	// Second upload data

	req, err := http.NewRequest(
		http.MethodPut,
		styleURL+"/"+url.PathEscape(entry.Name),
		strings.NewReader(data))
	if err != nil {
		return err
	}
	auth(req)
	if isSymbologyEncoding(data) {
		asContentType(req, "application/vnd.ogc.se+xml")
	} else {
		asContentType(req, "application/vnd.ogc.sld+xml")
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("cannot upload style %s (%s)",
			entry.Name, http.StatusText(resp.StatusCode))
	}

	// Third associate with layer

	if create {
		req, err := http.NewRequest(
			http.MethodPost,
			geoURL+"/rest/layers/"+
				url.PathEscape(workspaceName+":"+entry.Name)+
				"/styles?default=true",
			toStream(&styleFilename))
		if err != nil {
			return err
		}
		auth(req)
		asJSON(req)

		resp, err = http.DefaultClient.Do(req)
		if err != nil {
			return err
		}

		if resp.StatusCode != http.StatusCreated {
			return fmt.Errorf("cannot connect style %s with layer (%s)",
				entry.Name, http.StatusText(resp.StatusCode))
		}
	}

	return nil
}

// isSymbologyEncoding tries to figure out if its plain SLD or SE.
func isSymbologyEncoding(data string) bool {
	decoder := xml.NewDecoder(strings.NewReader(data))
	decoder.CharsetReader = charset.NewReaderLabel

	for {
		tok, err := decoder.Token()
		switch {
		case tok == nil && err == io.EOF:
			return false
		case err != nil:
			log.Printf("warn: invalid XML: %v\n", err)
			return false
		}
		if t, ok := tok.(xml.StartElement); ok &&
			t.Name.Space == "http://www.opengis.net/se" {
			return true
		}
	}
}

func ensureStyles() error {
	log.Println("info: creating styles")

	var stls styles
	if err := stls.load(); err != nil {
		return err
	}

	entries := models.InternalServices.Filter(
		models.IntAnd(
			models.IntWMS,
			models.IntWithStyle))

	var already []string

	defer func() {
		if len(already) > 0 {
			log.Printf("info: already having styles: %s\n",
				strings.Join(already, ", "))
		}
	}()

	for i := range entries {
		entry := &entries[i]
		if stls.hasStyle(entry.Name) {
			already = append(already, entry.Name)
			continue
		}
		if err := updateStyle(entry, true); err != nil {
			return err
		}
	}

	return nil
}

// PrepareGeoServer sets up the GeoServer to work together with the gemma server.
// It sets up a workspace, a datastore and exposes the features and styles.
func PrepareGeoServer() error {

	if config.DBUser() == "" {
		log.Println("info: Need metamorphic db user to configure GeoServer")
		return nil
	}

	if config.GeoServerURL() == "" {
		log.Println("info: No URL to GeoServer configured")
		return nil
	}

	for _, ensure := range []func() error{
		deleteWorkspace,
		ensureWorkspace,
		ensureDataStore,
		ensureFeatures,
		ensureStyles,
	} {
		if err := ensure(); err != nil {
			return err
		}
	}

	return nil
}

// ReconfigureStyle returns a function to update a style
// in the GeoServer to be in sync with the database.
func ReconfigureStyle(name string) {
	Reconfigure(func() error {
		var stls styles
		if err := stls.load(); err != nil {
			return err
		}

		entries := models.InternalServices.Filter(
			models.IntAnd(
				models.IntWMS,
				models.IntWithStyle,
				models.IntByName(name)))

		for i := range entries {
			entry := &entries[i]
			create := !stls.hasStyle(entry.Name)
			if err := updateStyle(entry, create); err != nil {
				return err
			}
		}

		return nil
	})
}