Skip to content

Commit 20d57a9

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #83 from metwork-framework/issue19
[WIP] feat: introduce webdav support for mfbase plugins
2 parents e5581b1 + 7301695 commit 20d57a9

27 files changed

+783
-112
lines changed

‎.layerapi2_dependencies‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ python3@mfcom
33
scientific_core@mfext
44
#+python3_circus@mfext
55
#+rpm@mfext
6+
openresty@mfext

‎adm/Makefile‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
BINS=_wait_postgresql.py before_start_postgresql _plugins.postinstall _plugins.preuninstall plugins.pgrestore.sh plugins.pgdump.sh _fix_plugin_owner.py _create_plugin_database.sh
1+
BINS=_wait_postgresql.py before_start_postgresql _plugins.postinstall _plugins.preuninstall plugins.pgrestore.sh plugins.pgdump.sh _fix_plugin_owner.py _create_plugin_database.sh before_start_nginx _make_nginx_conf __make_nginx_conf mfbase_conf_monitor.py
22
SHARES=sqltools.sh
33

44
include root.mk
@@ -8,4 +8,9 @@ include $(MFCOM_HOME)/share/startup_scripts_profiles.mk
88
TEMPLATES=$(shell find templates ! -type d)
99
TARGET_TEMPLATES:=$(addprefix $(PREFIX)/share/,$(TEMPLATES))
1010

11-
all:: profiles $(PREFIX)/bin/cronwrap.sh $(PREFIX)/bin/mfbase.status $(PREFIX)/bin/mfbase.start $(PREFIX)/bin/mfbase.stop $(PREFIX)/bin/mfbase.init load_env $(TARGET_TEMPLATES)
11+
all:: profiles $(PREFIX)/bin/cronwrap.sh $(PREFIX)/bin/mfbase.status $(PREFIX)/bin/mfbase.start $(PREFIX)/bin/mfbase.stop $(PREFIX)/bin/mfbase.init load_env $(TARGET_TEMPLATES) $(PREFIX)/share/welcome/index.html
12+
13+
$(PREFIX)/share/welcome/index.html: welcome/index.html
14+
rm -Rf $(PREFIX)/share/welcome
15+
mkdir -p $(PREFIX)/share/welcome
16+
cp -Rf welcome/* $(PREFIX)/share/welcome/

‎adm/__make_nginx_conf‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env python3
2+
3+
import os
4+
import glob
5+
import envtpl
6+
import sys
7+
from mflog import get_logger
8+
from mfutil.plugins import layerapi2_label_to_plugin_name
9+
from configparser_extended import ExtendedConfigParser
10+
11+
MODULE_RUNTIME_HOME = os.environ["MODULE_RUNTIME_HOME"]
12+
MFBASE_PLUGINS_HOME = os.path.join(MODULE_RUNTIME_HOME,
13+
"var", "plugins")
14+
CONFIG = os.environ.get('MFCONFIG', 'GENERIC')
15+
HOSTNAME = os.environ.get('MFCOM_HOSTNAME')
16+
HOSTNAME_FULL = os.environ.get('MFCOM_HOSTNAME_FULL')
17+
MODULE = os.environ['MODULE']
18+
19+
LOGGER = get_logger("__make_nginx_conf")
20+
21+
22+
def get_conf(plugin_configuration_file):
23+
plugin_conf = {}
24+
plugin_directory = os.path.dirname(plugin_configuration_file)
25+
parser = ExtendedConfigParser(config=CONFIG, strict=False,
26+
inheritance='im', interpolation=None)
27+
try:
28+
with open(os.path.join(plugin_directory,
29+
".layerapi2_label"), "r") as f:
30+
label = f.read().strip()
31+
except Exception:
32+
LOGGER.warning(
33+
"can't read %s/.layerapi2_label => ignoring this plugin",
34+
plugin_directory)
35+
return None
36+
try:
37+
plugin_name = layerapi2_label_to_plugin_name(label)
38+
except Exception as e:
39+
LOGGER.warning("can't read %s/.layerapi2_label with error: %s => "
40+
"ignoring this plugin", plugin_directory, e)
41+
return None
42+
plugin_conf['directory'] = plugin_directory
43+
plugin_conf['name'] = plugin_name
44+
parser.read(plugin_configuration_file)
45+
plugin_conf['use_postgresql'] = "1"
46+
if parser.has_option('general', 'use_postgresql'):
47+
if parser.getint("general", "use_postgresql") == 0:
48+
plugin_conf['use_postgresql'] = "0"
49+
plugin_conf['use_storage'] = "1"
50+
if parser.has_option('general', 'use_storage'):
51+
if parser.getint("general", "use_storage") == 0:
52+
plugin_conf['use_storage'] = "0"
53+
return plugin_conf
54+
55+
56+
plugins = []
57+
if len(sys.argv) == 2:
58+
config_files = glob.glob(sys.argv[1] + "/config.ini")
59+
else:
60+
config_files = glob.glob(MFBASE_PLUGINS_HOME + "/*/config.ini")
61+
for config_file in config_files:
62+
plugin_conf = get_conf(config_file)
63+
if plugin_conf:
64+
plugins.append(plugin_conf)
65+
66+
nginx_conf_file = os.path.join(os.environ['MODULE_HOME'], 'config',
67+
'nginx.conf')
68+
69+
with open(nginx_conf_file, "r") as f:
70+
extra_variables = {
71+
"PLUGINS": plugins
72+
}
73+
content = envtpl.render_string(f.read(), extra_variables=extra_variables,
74+
keep_multi_blank_lines=False)
75+
76+
print(content)

‎adm/_make_nginx_conf‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
set -eu
4+
5+
TMPFILE="${MODULE_RUNTIME_HOME}/tmp/nginx_conf.$$"
6+
TMPFILE2="${MODULE_RUNTIME_HOME}/tmp/nginx_conf2.$$"
7+
TMPFILE3="${MODULE_RUNTIME_HOME}/tmp/nginx_conf3.$$"
8+
9+
UUID=$(cat "${MODULE_RUNTIME_HOME}/var/uuid" 2>/dev/null)
10+
if test "${UUID}" = ""; then
11+
UUID="unknown"
12+
fi
13+
export UUID
14+
15+
__make_nginx_conf "$@" >"${TMPFILE}"
16+
nginxfmt.py "${TMPFILE}"
17+
# FIXME: ugly hack to circumvent nginxfmt problem with JSON
18+
cat -s "${TMPFILE}" |sed 's/~~~1/{/g' |sed 's/~~~2/}/g' |grep -v 'FIXME: ugly hack' >"${TMPFILE2}"
19+
rm -f "${TMPFILE}"
20+
21+
if ! test -f "${MODULE_RUNTIME_HOME}/tmp/config_auto/mime.types"; then
22+
cp -f "${MFEXT_HOME}/opt/openresty/config/mime.types" "${MODULE_RUNTIME_HOME}/tmp/config_auto/mime.types"
23+
fi
24+
set +e
25+
"${MFEXT_HOME}/opt/openresty/nginx/sbin/nginx" -t -c "${TMPFILE2}" >"${TMPFILE3}" 2>&1
26+
if test $? -ne 0; then
27+
>&2 echo "ERROR: bad nginx configuration (${TMPFILE2} => see ${TMPFILE3} for details"
28+
exit 1
29+
fi
30+
set -e
31+
32+
cat "${TMPFILE2}"
33+
rm -f "${TMPFILE2}" "${TMPFILE3}"

‎adm/_plugins.postinstall‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,25 +24,47 @@ if ! test -d "${PLUGIN_HOME}"; then
2424
exit 0
2525
fi
2626

27-
source "${MODULE_HOME}/share/sqltools.sh"
28-
29-
echo -n "- Create user plugin_${NAME}..."
30-
echo_running
31-
createuser -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" --no-createdb --no-createrole --no-superuser "plugin_${NAME}" >/tmp/createuser.$$ 2>&1
32-
if test $? -eq 0; then
33-
echo_ok
34-
rm -f /tmp/createuser.$$
27+
T=$(config.py "${PLUGIN_HOME}/config.ini" general use_postgresql 2>/dev/null)
28+
if test "${T}" = "0"; then
29+
USE_POSTGRESQL=0
3530
else
36-
echo_nok
37-
echo_bold "see /tmp/createuser.$$ for more details"
38-
exit 1
31+
USE_POSTGRESQL=1
32+
fi
33+
T=$(config.py "${PLUGIN_HOME}/config.ini" general use_storage 2>/dev/null)
34+
if test "${T}" = "0"; then
35+
USE_STORAGE=0
36+
else
37+
USE_STORAGE=1
3938
fi
4039

41-
_create_plugin_database.sh "${NAME}"
40+
if test "${USE_STORAGE}" = "1"; then
41+
mkdir -p "${MFBASE_NGINX_STORAGE_DIR}"
42+
if test -d "${PLUGIN_HOME}/initial_storage"; then
43+
rm -Rf "${MFBASE_NGINX_STORAGE_DIR:?}/${NAME}"
44+
cp -Rf "${PLUGIN_HOME}/initial_storage" "${MFBASE_NGINX_STORAGE_DIR}/${NAME}"
45+
fi
46+
fi
47+
if test "${USE_POSTGRESQL}" = "1"; then
48+
source "${MODULE_HOME}/share/sqltools.sh"
4249

43-
SQLFILES=$(ls ${PLUGIN_HOME}/sql/*.sql 2>/dev/null)
50+
echo -n "- Create postgresql user plugin_${NAME}..."
51+
echo_running
52+
createuser -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" --no-createdb --no-createrole --no-superuser "plugin_${NAME}" >/tmp/createuser.$$ 2>&1
53+
if test $? -eq 0; then
54+
echo_ok
55+
rm -f /tmp/createuser.$$
56+
else
57+
echo_nok
58+
echo_bold "see /tmp/createuser.$$ for more details"
59+
exit 1
60+
fi
4461

45-
for SQLFILE in ${SQLFILES}; do
46-
BASENAME=$(basename "${SQLFILE}")
47-
batch_psql "${SQLFILE}" "Playing ${BASENAME}" "plugin_${NAME}" "plugin_${NAME}" || exit 1
48-
done
62+
_create_plugin_database.sh "${NAME}"
63+
64+
SQLFILES=$(ls ${PLUGIN_HOME}/sql/*.sql 2>/dev/null)
65+
66+
for SQLFILE in ${SQLFILES}; do
67+
BASENAME=$(basename "${SQLFILE}")
68+
batch_psql "${SQLFILE}" "Playing ${BASENAME}" "plugin_${NAME}" "plugin_${NAME}" || exit 1
69+
done
70+
fi

‎adm/_plugins.preuninstall‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,20 +28,38 @@ source "${MODULE_HOME}/share/sqltools.sh"
2828

2929
RES=0
3030

31-
# see https://dba.stackexchange.com/questions/11893/force-drop-db-while-others-may-be-connected
32-
# see https://github.com/metwork-framework/mfbase/issues/23
33-
cat >/tmp/drop.$$ <<EOF
31+
T=$(config.py "${PLUGIN_HOME}/config.ini" general use_postgresql 2>/dev/null)
32+
if test "${T}" = "0"; then
33+
USE_POSTGRESQL=0
34+
else
35+
USE_POSTGRESQL=1
36+
fi
37+
T=$(config.py "${PLUGIN_HOME}/config.ini" general use_storage 2>/dev/null)
38+
if test "${T}" = "0"; then
39+
USE_STORAGE=0
40+
else
41+
USE_STORAGE=1
42+
fi
43+
44+
if test "${USE_STORAGE}" = "1"; then
45+
rm -Rf "${MFBASE_NGINX_STORAGE_DIR:?}/${NAME}"
46+
fi
47+
if test "${USE_POSTGRESQL}" = "1"; then
48+
# see https://dba.stackexchange.com/questions/11893/force-drop-db-while-others-may-be-connected
49+
# see https://github.com/metwork-framework/mfbase/issues/23
50+
cat >/tmp/drop.$$ <<EOF
3451
UPDATE pg_database SET datallowconn = 'false' WHERE datname = 'plugin_${NAME}';
3552
EOF
36-
batch_psql /tmp/drop.$$ "make sure no one can connect to this database" || RES=1
37-
rm -f /tmp/drop.$$
53+
batch_psql /tmp/drop.$$ "make sure no one can connect to this database" || RES=1
54+
rm -f /tmp/drop.$$
3855

39-
cat >/tmp/drop.$$ <<EOF
56+
cat >/tmp/drop.$$ <<EOF
4057
SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'plugin_${NAME}';
4158
EOF
42-
batch_psql /tmp/drop.$$ "force disconnection of all clients connected" || RES=1
43-
rm -f /tmp/drop.$$
59+
batch_psql /tmp/drop.$$ "force disconnection of all clients connected" || RES=1
60+
rm -f /tmp/drop.$$
4461

45-
dropdb -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" "plugin_${NAME}" || RES=1
46-
dropuser -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" "plugin_${NAME}" || RES=1
62+
dropdb -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" "plugin_${NAME}" || RES=1
63+
dropuser -h "${MODULE_RUNTIME_HOME}/var" -p "${MFBASE_POSTGRESQL_PORT}" -U "${MFBASE_POSTGRESQL_USERNAME}" "plugin_${NAME}" || RES=1
64+
fi
4765
exit ${RES}

‎adm/before_start_nginx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
mkdir -p "${MFBASE_NGINX_STORAGE_DIR}"
4+
PLUGINS=$(plugins.list --raw 2>/dev/null |awk -F '~~~' '{print $1;}')
5+
for PLUGIN in ${PLUGINS}; do
6+
H=$(get_layer_home "plugin_${PLUGIN}@${MODULE_LOWERCASE}")
7+
T=$(config.py "${H}/config.ini" general use_storage 2>/dev/null)
8+
if test "${T}" = "0"; then
9+
USE_STORAGE=0
10+
else
11+
USE_STORAGE=1
12+
fi
13+
if test "${USE_STORAGE}" = "1"; then
14+
mkdir -p "${MFBASE_NGINX_STORAGE_DIR}/${PLUGIN}"
15+
fi
16+
done
17+
"${MFCOM_HOME}/bin/before_start_nginx"

‎adm/mfbase_conf_monitor.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/usr/bin/env python3
2+
3+
import os
4+
import time
5+
from mflog import getLogger
6+
from mfutil import BashWrapper, BashWrapperOrRaise
7+
from conf_monitor import ConfMonitorRunner, md5sumfile
8+
9+
LOGGER = getLogger("conf_monitor")
10+
MODULE_RUNTIME_HOME = os.environ['MODULE_RUNTIME_HOME']
11+
NGINX_FLAG = (int(os.environ['MFBASE_NGINX_FLAG']) == 1)
12+
13+
14+
def make_new_nginx_conf():
15+
new_nginx_conf = "%s/tmp/tmp_nginx_conf2" % MODULE_RUNTIME_HOME
16+
cmd = "_make_nginx_conf >%s" % new_nginx_conf
17+
BashWrapperOrRaise(cmd)
18+
return (new_nginx_conf, md5sumfile(new_nginx_conf))
19+
20+
21+
def get_old_nginx_conf():
22+
old_nginx_conf = "%s/tmp/config_auto/nginx.conf" % MODULE_RUNTIME_HOME
23+
return (old_nginx_conf, md5sumfile(old_nginx_conf))
24+
25+
26+
def restart_nginx(old_conf, new_conf):
27+
os.unlink(old_conf)
28+
os.rename(new_conf, old_conf)
29+
x = BashWrapper("_nginx.reload")
30+
if not x:
31+
LOGGER.warning(x)
32+
33+
34+
class MfbaseConfMonitorRunner(ConfMonitorRunner):
35+
36+
def manage_nginx(self):
37+
if not NGINX_FLAG:
38+
return True
39+
new_conf, new_md5 = make_new_nginx_conf()
40+
old_conf, old_md5 = get_old_nginx_conf()
41+
if new_md5 != old_md5:
42+
LOGGER.info("nginx conf changed => restart nginx...")
43+
restart_nginx(old_conf, new_conf)
44+
time.sleep(3)
45+
else:
46+
LOGGER.debug("nginx conf didn't change")
47+
return True
48+
49+
def handle_event(self):
50+
return self.manage_nginx() and self.manage_crontab() and \
51+
self.manage_circus()
52+
53+
54+
if __name__ == '__main__':
55+
x = MfbaseConfMonitorRunner()
56+
x.run()

0 commit comments

Comments
 (0)