Mercurial > gemma

comparison schema/geonames-import/import-geonames.sh @ 1109:74a75a5ce770

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added geonames data and extended search for villages/cities. The free data from geonames https://download.geonames.org/ is imported into the gemma data base (for the relevant states) and used to search for cities and villages. This data might be replaced by data from the RIS-index later on..?
author Sascha Wilde <wilde@intevation.de>
date Mon, 05 Nov 2018 13:07:16 +0100
parents
children ca7821e1f720
comparison
equal deleted inserted replaced
1108:5adee8b0e29f 1109:74a75a5ce770
1 #!/bin/bash
2
3 # This is Free Software under GNU Affero General Public License v >= 3.0
4 # without warranty, see README.md and license for details.
5
6 # SPDX-License-Identifier: AGPL-3.0-or-later
7 # License-Filename: LICENSES/AGPL-3.0.txt
8
9 # Copyright (C) 2018 by via donau
10 # – Österreichische Wasserstraßen-Gesellschaft mbH
11 # Software engineering by Intevation GmbH
12
13 # Author(s):
14 # * Sascha Wilde <wilde@intevation.de>
15
16
17 ME=$(basename $0)
18 MYHOME=$(realpath $(dirname $0))
19 CCs="AT BG HR HU RO SK"
20
21 # Defaults:
22 datadir=$(realpath "$MYHOME/data")
23 db=gemma
24 port=5432
25 drop=0
26
27 usage()
28 {
29 cat <<EOF
30 $ME [OPTION]...
31
32 Import geonames data (used for extended search features) into gemma database.
33
34 If data was downloaded previously, it will be reused. To force update
35 of data, first delete the existing data directory or use a fresh one.
36
37 Options:
38 -d, --db=NAME create the database NAME. Default: "gemma"
39 -p, --port=PORT connect do the postgresql cluster at PORT.
40 Default is the postgresql standard port 5432
41 -D, --datadir=PATH download data to PATH. Default: "$datadir"
42 --drop drop geonames data
43 --help display this help and exit
44 EOF
45 }
46
47 fatal()
48 {
49 echo >&2 "$1"
50 exit 23
51 }
52
53 dl() {
54 [ -d "$datadir" ] || mkdir "$datadir"
55 cd "$datadir"
56 for cc in $CCs ; do
57 if [ -f "${cc}.zip" ] ; then
58 echo "Keeping existing ${cc}.zip ..."
59 else
60 echo "Fetching ${cc}.zip ..."
61 curl -O "https://download.geonames.org/export/dump/${cc}.zip"
62 fi
63 done
64 }
65
66 unpack()
67 {
68 cd "$datadir" || fatal "$datadir not missing!"
69 for cc in $CCs ; do
70 if [ -f "${cc}.zip" ] ; then
71 if [ -f "${cc}.txt" ] ; then
72 echo "Reusing existing ${cc}.txt"
73 else
74 echo "Extracting ${cc}.zip ..."
75 unzip "${cc}.zip" "${cc}.txt"
76 fi
77 else
78 fatal "Missing archive: ${cc}.zip"
79 fi
80 done
81 }
82
83 drop_data()
84 {
85 echo "Dropping geonames data..."
86 psql -q -p "$port" -d "$db" -c "TRUNCATE TABLE waterway.geonames;"
87 }
88
89 import_data()
90 {
91 for cc in $CCs ; do
92 fqfn="${datadir}/${cc}.txt"
93 echo "Importing $fqfn into database..."
94 psql -qtv ON_ERROR_STOP= -p "$port" -d "$db" \
95 -v filename="$fqfn" \
96 -f "$MYHOME/import-geonames.sql"
97 done
98 }
99
100 # Parse options:
101
102 OPTS=`getopt \
103 -l help,db:,port:,datadir:,drop \
104 -o D:d:p: -n "$ME" -- "$@"`
105 [ $? -eq 0 ] || { usage ; exit 1 ; }
106
107 eval set -- "$OPTS"
108
109 while true ; do
110 case "$1" in
111 --db|-d)
112 db="$2"
113 shift 2
114 ;;
115 --port|-p)
116 port="$2"
117 shift 2
118 ;;
119 --datadir|-D)
120 datadir="$2"
121 shift 2
122 ;;
123 --drop)
124 drop=1
125 shift 1
126 ;;
127 --help)
128 { usage ; exit 0 ; }
129 ;;
130 --)
131 shift
132 break
133 ;;
134 esac
135 done
136
137
138 # Main ------------------------------------------------------------
139
140 if [ $drop -eq 0 ] ; then
141 dl
142 unpack
143 drop_data
144 import_data
145 else
146 drop_data
147 fi