caddy and maxmind updates

Dockerfile

@@ -23,7 +23,7 @@ RUN make install
 WORKDIR /goaccess
 RUN cp /goaccess-temp/goaccess.tar.gz .
 RUN tar --strip-components=1  -xzvf goaccess.tar.gz
-RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.30<\/span> )/" /goaccess/resources/tpls.html
+RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( <a href='https:\/\/tiny.one\/xgoan'>GOAN<\/a> <span>v1.1.31<\/span> )/" /goaccess/resources/tpls.html
 RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css
 RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline
 RUN make

README.md

@@ -12,6 +12,7 @@ Still in development... You might need to wait a bit if you have a large amount
 - NPM Redirection
 - NPM Error
 - Traefik
+- Caddy
 - Load your own custom config as well to parse other logs
 
 <br/>
@@ -111,8 +112,8 @@ services:
 | `-e BASIC_AUTH_PASSWORD=pass`         |   (Optional) Requires BASIC_AUTH to bet set to True.  Password for basic authentication.     |
 | `-e EXCLUDE_IPS=`         |   (Optional) IP Addresses or range of IPs delimited by comma refer to https://goaccess.io/man. For example: 192.168.0.1-192.168.0.100 or 127.0.0.1,192.168.0.1-192.168.0.100   |
 | `-e INCLUDE_PROXY_HOSTS=`         |   (Optional) Only consume the list of provided proxy hosts. This is a comma separated list containing the proxy host number for example "11,21". This would consume proxy-host-11_access.log* and proxy-host-21_access.log* . The host number can be found right clicking on the 3 dots on the proxy host line in the GUI.  |
-| `-e LOG_TYPE=`         |   (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED. More information below.|
-| `-e LOG_TYPE_FILE_PATTERN=`         |   (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! |
+| `-e LOG_TYPE=`         |   (Optional) By default the configuration will be set to read NPM logs. Options are: CUSTOM, NPM, NPM+R, TRAEFIK, NCSA_COMBINED, CADDY_V1. More information below.|
+| `-e LOG_TYPE_FILE_PATTERN=`         |   (Optional) Only to be used with LOG_TYPE=NCSA_COMBINED, CADDY_V1, or TRAEFIK. This parameter will pass along the file type you are trying match. For example you can pass -e LOG_TYPE_FILE_PATTERN="*.log" or -e LOG_TYPE_FILE_PATTERN="access.log". The default is *.log. Please keep it simple as I have not tested this completely. Use at your own RISK! |
 | `-e LANG=zh_CN.UTF-8 -e LANGUAGE=zh_CN.UTF-8`         |   (Optional) Language localization added. GoAccess only has a few translations available. Please visit https://github.com/allinurl/goaccess/tree/master/po to see the translations available. <br/><br/>**Current Translations**<br/>de - German<br/>es - Spanish<br/>fr - French<br/>it - Italian<br/>ja - Japanese<br/>ko - Korean<br/>pt_BR - Portuguese (Brazil)<br/>ru - Russian<br/>sv - Swedish<br/>uk - English (United Kingdom)<br/>zh_CN - Chinese - Simplified|
 | `-e ENABLE_BROWSERS_LIST=True/False`         |   (Optional) Defaults to False. Set to true if you would like to enable the [goaccess browsers.list](https://github.com/allinurl/goaccess/blob/master/config/browsers.list) file.     |
 | `-e CUSTOM_BROWSERS=`         |   - (Optional) Consumes the list of provided custom browsers. This is a comma separated list containing the custom browser(s) in the format `Browser:Browser_category`.<br/>- If your custom browser is already defined in the default `browsers.list` file, it will not be added. However, the `Browser_category` can be reused.<br/><br/> CUSTOM_BROWSERS list example: `Kuma:Crawlers,TestBrowser:Crawlers,Kuma:Uptime,Discordbot:Crawlers`<br/><br/>For the example above, only `Kuma:Crawlers` and `TestBrowser:Crawlers` will be appended to the `browsers.list` file. <br/><br/>`Kuma:Uptime` is ignored as the browser `Kuma` has already been defined in `Kuma:Crawlers`. `Discordbot:Crawlers` is ignored as the browser `Discordbot` is already defined in the [default browsers.list file](https://github.com/allinurl/goaccess/blob/master/config/browsers.list)<br/><br/>Note for users using CUSTOM LOG_TYPE:<br/><br/>If your `goaccess.conf` file references a browsers.list file other than the one located in the `/goaccess-config/ directory`, the CUSTOM_BROWSERS variable will be ignored.    |
@@ -170,7 +171,11 @@ services:
     - You can also set the following environment variables yourself to override the defaults
       - TIME_FORMAT
       - DATE_FORMAT
-      - LOG_FORMAT  
+      - LOG_FORMAT 
+  - CADDY_V1
+    - environment parameters that will not work and will be ignored
+      - SKIP_ARCHIVED_LOGS
+    - by default only access.log is read and parsed
 
       ```
       FOR EXAMPLE
@@ -226,6 +231,13 @@ date-format %d/%b/%Y
 log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u"
 ```
 
+### CADDY LOG FORMAT
+```
+time-format %s
+date-format %s
+date-spec min
+log-format CADDY
+```
 
 # **Possible/Known Issues** 
 - A lot of CPU Usage and 10000 request every second in webUI

resources/scripts/logs/caddy.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+function caddyV1_init(){
+    goan_config="/goaccess-config/goaccess.conf"
+    nginx_html="/var/www/html/index.html"
+    html_config="/var/www/html/goaccess_conf.html"
+    archive_log="/goaccess-config/archive.log"
+    active_log="/goaccess-config/active.log"
+
+    if [[ -f ${goan_config} ]]; then
+        rm ${goan_config}
+    else
+        mkdir -p "/goaccess-config/"
+        cp /goaccess-config/goaccess.conf.bak ${goan_config}
+    fi
+    if [[ -f ${nginx_html} ]]; then
+        rm ${nginx_html}
+    else
+        mkdir -p "/var/www/html/"
+        touch ${nginx_html}
+    fi
+    if [[ -f ${html_config} ]]; then
+        rm ${html_config}
+    fi
+
+    echo -n "" > ${archive_log}
+    echo -n "" > ${active_log}
+}
+
+function caddyV1_goaccess_config(){
+    echo -e "\n\n\n" >> ${goan_config}
+    echo "######################################" >> ${goan_config}
+    echo "# ${goan_version}" >> ${goan_config}
+    echo "# GOAN_PROXY_CONFIG" >> ${goan_config}
+    echo "######################################" >> ${goan_config}
+    echo "time-format ${TIME_FORMAT:-%s}" >> ${goan_config}
+    echo "date-format ${DATE_FORMAT:-%s}" >> ${goan_config}
+    echo "date-spec min" >> ${goan_config}
+    echo "log-format CADDY" >> ${goan_config}
+    echo "port 7890" >> ${goan_config}
+    echo "real-time-html true" >> ${goan_config}
+    echo "output ${nginx_html}" >> ${goan_config}
+    if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then
+        echo -e "\n\tENABLING CADDY INSTANCE GOACCESS BROWSERS LIST"
+        browsers_file="/goaccess-config/browsers.list"
+        echo "browsers-file ${browsers_file}" >> ${goan_config}
+    fi
+}
+
+function caddyV1(){
+    caddyV1_init
+    caddyV1_goaccess_config
+
+    echo -e "\nLOADING CADDY LOGS"
+    echo "-------------------------------"
+
+    echo $'\n' >> ${goan_config}
+    echo "#GOAN_CADDY_LOG_FILES" >> ${goan_config}
+    echo "log-file ${archive_log}" >> ${goan_config}
+    echo "log-file ${active_log}" >> ${goan_config}
+
+    goan_log_count=0
+    goan_archive_log_count=0
+
+    echo -e "\n#GOAN_PROXY_FILES" >> ${goan_config}
+    if [[ -d "${goan_log_path}" ]]; then
+
+        echo -e "\n\tAdding proxy logs..."
+        IFS=$'\n'
+
+        if [[ -z "${LOG_TYPE_FILE_PATTERN}" ]]; then
+            LOG_TYPE_FILE_PATTERN="access.log"
+        else
+            LOG_TYPE_FILE_PATTERN="*.log"
+        fi
+
+        for file in $(find "${goan_log_path}" -name "${LOG_TYPE_FILE_PATTERN}");
+        do
+            if [ -f $file ]
+            then
+                if [ -r $file ] && R="Read = yes" || R="Read = No"
+                then
+                    echo "log-file ${file}" >> ${goan_config}
+                    goan_log_count=$((goan_log_count+1))
+                    echo -ne ' \t '
+                    echo "Filename: $file | $R"
+                else
+                    echo -ne ' \t '
+                    echo "Filename: $file | $R"
+                fi
+            else
+                echo -ne ' \t '
+                echo "Filename: $file | Not a file"
+            fi
+        done
+        unset IFS
+    else
+        echo "Problem loading directory (check directory or permissions)... ${goan_log_path}"
+    fi
+
+    if [ $goan_log_count != 0 ]
+    then
+        echo "Found (${goan_log_count}) proxy logs..."
+    else
+        echo "No access.log found. Creating an empty log file..."
+        touch "${goan_log_path}/access.log"
+    fi
+
+    #additonal config settings
+    exclude_ips             ${goan_config}
+    debug                   ${goan_config} ${html_config}
+    set_geoip_database      ${goan_config}
+
+    echo -e "\nSKIP ARCHIVED LOGS"
+    echo "-------------------------------"
+    echo "FEATURE NOT AVAILABLE FOR CADDY"
+
+    #write out loading page
+    echo "<!doctype html><html><head>" > ${nginx_html}
+    echo "<title>GOAN - ${goan_version}</title>" >> ${nginx_html}
+    echo "<meta http-equiv=\"refresh\" content=\"1\" >" >> ${nginx_html}
+    echo "<style>body {font-family: Arial, sans-serif;}</style>" >> ${nginx_html}
+    echo "</head><body><p><b>${goan_version}</b><br/><br/>loading... <br/><br/>" >> ${nginx_html}
+    echo "Logs processing: $(($goan_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html}
+    echo "<br/></p></body></html>" >> ${nginx_html}
+
+    echo -e "\nRUN CADDY GOACCESS"
+    runGoAccess
+}

resources/scripts/start.sh

@@ -8,8 +8,9 @@ source $(dirname "$0")/logs/traefik.sh
 source $(dirname "$0")/logs/custom.sh
 source $(dirname "$0")/logs/ncsa_combined.sh
 source $(dirname "$0")/logs/nginx_access.sh
+source $(dirname "$0")/logs/caddy.sh
 
-goan_version="GOAN v1.1.30"
+goan_version="GOAN v1.1.31"
 goan_log_path="/opt/log"
 
 goaccess_ping_interval=15
@@ -83,6 +84,8 @@ elif [[ "${LOG_TYPE}" == "CUSTOM" ]]; then
     custom
 elif [[ "${LOG_TYPE}" == "NGINX_ACCESS" ]]; then
     nginx_access
+elif [[ "${LOG_TYPE}" == "CADDY_V1" ]]; then
+    caddyV1 
 fi
 # END PROXY LOGS