diff --git a/Dockerfile b/Dockerfile index 3735438..cf01f73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN make install WORKDIR /goaccess RUN cp /goaccess-temp/goaccess.tar.gz . RUN tar --strip-components=1 -xzvf goaccess.tar.gz -RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( GOAN<\/a> v1.1.30<\/span> )/" /goaccess/resources/tpls.html +RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( GOAN<\/a> v1.1.31<\/span> )/" /goaccess/resources/tpls.html RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline RUN make diff --git a/README.md b/README.md index 08cb87c..aceaf84 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Still in development... You might need to wait a bit if you have a large amount - NPM Redirection - NPM Error - Traefik +- Caddy - Load your own custom config as well to parse other logs
@@ -111,8 +112,8 @@ services: | `-e BASIC_AUTH_PASSWORD=pass` | (Optional) Requires BASIC_AUTH to bet set to True. Password for basic authentication. | | `-e EXCLUDE_IPS=` | (Optional) IP Addresses or range of IPs delimited by comma refer to https://goaccess.io/man. For example: 192.168.0.1-192.168.0.100 or 127.0.0.1,192.168.0.1-192.168.0.100 | | `-e INCLUDE_PROXY_HOSTS=` | (Optional) Only consume the list of provided proxy hosts. This is a comma separated list containing the proxy host number for example "11,21". This would consume proxy-host-11_access.log* and proxy-host-21_access.log* . The host number can be found right clicking on the 3 dots on the proxy host line in the GUI. | -| `-e LOG_TYPE=` | (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED. More information below.| -| `-e LOG_TYPE_FILE_PATTERN=` | (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! | +| `-e LOG_TYPE=` | (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED, CADDY_V1. More information below.| +| `-e LOG_TYPE_FILE_PATTERN=` | (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED, CADDY_V1, or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! | | `-e LANG=zh_CN.UTF-8 -e LANGUAGE=zh_CN.UTF-8` | (Optional) Language localization added. GoAccess only has a few translations available. Please visit https://github.com/allinurl/goaccess/tree/master/po to see the translations available.

**Current Translations**
de - German
es - Spanish
fr - French
it - Italian
ja - Japanese
ko - Korean
pt_BR - Portuguese (Brazil)
ru - Russian
sv - Swedish
uk - English (United Kingdom)
zh_CN - Chinese - Simplified| | `-e ENABLE_BROWSERS_LIST=True/False` | (Optional) Defaults to False. Set to true if you would like to enable the [goaccess browsers.list](https://github.com/allinurl/goaccess/blob/master/config/browsers.list) file. | | `-e CUSTOM_BROWSERS=` | - (Optional) Consumes the list of provided custom browsers. This is a comma separated list containing the custom browser(s) in the format `Browser:Browser_category`.
- If your custom browser is already defined in the default `browsers.list` file, it will not be added. However, the `Browser_category` can be reused.

CUSTOM_BROWSERS list example: `Kuma:Crawlers,TestBrowser:Crawlers,Kuma:Uptime,Discordbot:Crawlers`

For the example above, only `Kuma:Crawlers` and `TestBrowser:Crawlers` will be appended to the `browsers.list` file.

`Kuma:Uptime` is ignored as the browser `Kuma` has already been defined in `Kuma:Crawlers`. `Discordbot:Crawlers` is ignored as the browser `Discordbot` is already defined in the [default browsers.list file](https://github.com/allinurl/goaccess/blob/master/config/browsers.list)

Note for users using CUSTOM LOG_TYPE:

If your `goaccess.conf` file references a browsers.list file other than the one located in the `/goaccess-config/ directory`, the CUSTOM_BROWSERS variable will be ignored. | @@ -170,7 +171,11 @@ services: - You can also set the following environment variables yourself to override the defaults - TIME_FORMAT - DATE_FORMAT - - LOG_FORMAT + - LOG_FORMAT + - CADDY_V1 + - environment parameters that will not work and will be ignored + - SKIP_ARCHIVED_LOGS + - by default only access.log is read and parsed ``` FOR EXAMPLE @@ -226,6 +231,13 @@ date-format %d/%b/%Y log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" ``` +### CADDY LOG FORMAT +``` +time-format %s +date-format %s +date-spec min +log-format CADDY +``` # **Possible/Known Issues** - A lot of CPU Usage and 10000 request every second in webUI diff --git a/assests/maxmind/GeoLite2-ASN.mmdb b/assests/maxmind/GeoLite2-ASN.mmdb index 86c962b..9581f0f 100644 Binary files a/assests/maxmind/GeoLite2-ASN.mmdb and b/assests/maxmind/GeoLite2-ASN.mmdb differ diff --git a/assests/maxmind/GeoLite2-City.mmdb b/assests/maxmind/GeoLite2-City.mmdb index fe2e742..e8f23c7 100644 Binary files a/assests/maxmind/GeoLite2-City.mmdb and b/assests/maxmind/GeoLite2-City.mmdb differ diff --git a/assests/maxmind/GeoLite2-Country.mmdb b/assests/maxmind/GeoLite2-Country.mmdb index 5f75fd4..c450878 100644 Binary files a/assests/maxmind/GeoLite2-Country.mmdb and b/assests/maxmind/GeoLite2-Country.mmdb differ diff --git a/resources/scripts/logs/caddy.sh b/resources/scripts/logs/caddy.sh new file mode 100644 index 0000000..cc34276 --- /dev/null +++ b/resources/scripts/logs/caddy.sh @@ -0,0 +1,128 @@ +#!/bin/bash +function caddyV1_init(){ + goan_config="/goaccess-config/goaccess.conf" + nginx_html="/var/www/html/index.html" + html_config="/var/www/html/goaccess_conf.html" + archive_log="/goaccess-config/archive.log" + active_log="/goaccess-config/active.log" + + if [[ -f ${goan_config} ]]; then + rm ${goan_config} + else + mkdir -p "/goaccess-config/" + cp /goaccess-config/goaccess.conf.bak ${goan_config} + fi + if [[ -f ${nginx_html} ]]; then + rm ${nginx_html} + else + mkdir -p "/var/www/html/" + touch ${nginx_html} + fi + if [[ -f ${html_config} ]]; then + rm ${html_config} + fi + + echo -n "" > ${archive_log} + echo -n "" > ${active_log} +} + +function caddyV1_goaccess_config(){ + echo -e "\n\n\n" >> ${goan_config} + echo "######################################" >> ${goan_config} + echo "# ${goan_version}" >> ${goan_config} + echo "# GOAN_PROXY_CONFIG" >> ${goan_config} + echo "######################################" >> ${goan_config} + echo "time-format ${TIME_FORMAT:-%s}" >> ${goan_config} + echo "date-format ${DATE_FORMAT:-%s}" >> ${goan_config} + echo "date-spec min" >> ${goan_config} + echo "log-format CADDY" >> ${goan_config} + echo "port 7890" >> ${goan_config} + echo "real-time-html true" >> ${goan_config} + echo "output ${nginx_html}" >> ${goan_config} + if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then + echo -e "\n\tENABLING CADDY INSTANCE GOACCESS BROWSERS LIST" + browsers_file="/goaccess-config/browsers.list" + echo "browsers-file ${browsers_file}" >> ${goan_config} + fi +} + +function caddyV1(){ + caddyV1_init + caddyV1_goaccess_config + + echo -e "\nLOADING CADDY LOGS" + echo "-------------------------------" + + echo $'\n' >> ${goan_config} + echo "#GOAN_CADDY_LOG_FILES" >> ${goan_config} + echo "log-file ${archive_log}" >> ${goan_config} + echo "log-file ${active_log}" >> ${goan_config} + + goan_log_count=0 + goan_archive_log_count=0 + + echo -e "\n#GOAN_PROXY_FILES" >> ${goan_config} + if [[ -d "${goan_log_path}" ]]; then + + echo -e "\n\tAdding proxy logs..." + IFS=$'\n' + + if [[ -z "${LOG_TYPE_FILE_PATTERN}" ]]; then + LOG_TYPE_FILE_PATTERN="access.log" + else + LOG_TYPE_FILE_PATTERN="*.log" + fi + + for file in $(find "${goan_log_path}" -name "${LOG_TYPE_FILE_PATTERN}"); + do + if [ -f $file ] + then + if [ -r $file ] && R="Read = yes" || R="Read = No" + then + echo "log-file ${file}" >> ${goan_config} + goan_log_count=$((goan_log_count+1)) + echo -ne ' \t ' + echo "Filename: $file | $R" + else + echo -ne ' \t ' + echo "Filename: $file | $R" + fi + else + echo -ne ' \t ' + echo "Filename: $file | Not a file" + fi + done + unset IFS + else + echo "Problem loading directory (check directory or permissions)... ${goan_log_path}" + fi + + if [ $goan_log_count != 0 ] + then + echo "Found (${goan_log_count}) proxy logs..." + else + echo "No access.log found. Creating an empty log file..." + touch "${goan_log_path}/access.log" + fi + + #additonal config settings + exclude_ips ${goan_config} + debug ${goan_config} ${html_config} + set_geoip_database ${goan_config} + + echo -e "\nSKIP ARCHIVED LOGS" + echo "-------------------------------" + echo "FEATURE NOT AVAILABLE FOR CADDY" + + #write out loading page + echo "" > ${nginx_html} + echo "GOAN - ${goan_version}" >> ${nginx_html} + echo "" >> ${nginx_html} + echo "" >> ${nginx_html} + echo "

${goan_version}

loading...

" >> ${nginx_html} + echo "Logs processing: $(($goan_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html} + echo "

" >> ${nginx_html} + + echo -e "\nRUN CADDY GOACCESS" + runGoAccess +} diff --git a/resources/scripts/start.sh b/resources/scripts/start.sh index 1d1c793..0d7c56b 100755 --- a/resources/scripts/start.sh +++ b/resources/scripts/start.sh @@ -8,8 +8,9 @@ source $(dirname "$0")/logs/traefik.sh source $(dirname "$0")/logs/custom.sh source $(dirname "$0")/logs/ncsa_combined.sh source $(dirname "$0")/logs/nginx_access.sh +source $(dirname "$0")/logs/caddy.sh -goan_version="GOAN v1.1.30" +goan_version="GOAN v1.1.31" goan_log_path="/opt/log" goaccess_ping_interval=15 @@ -83,6 +84,8 @@ elif [[ "${LOG_TYPE}" == "CUSTOM" ]]; then custom elif [[ "${LOG_TYPE}" == "NGINX_ACCESS" ]]; then nginx_access +elif [[ "${LOG_TYPE}" == "CADDY_V1" ]]; then + caddyV1 fi # END PROXY LOGS