From d48bb6525c5d5a0fa84bac9614d688f843a0beba Mon Sep 17 00:00:00 2001 From: Xavier Date: Sat, 20 Jan 2024 21:30:34 -0600 Subject: [PATCH 1/2] v1.1.25 - added parsing of the NGINX access.log --- Dockerfile | 2 +- README.md | 10 ++ resources/scripts/logs/nginx_access.sh | 121 +++++++++++++++++++++++++ resources/scripts/start.sh | 5 +- 4 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 resources/scripts/logs/nginx_access.sh diff --git a/Dockerfile b/Dockerfile index ee36a7e..7180571 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN make install WORKDIR /goaccess RUN cp /goaccess-temp/goaccess.tar.gz . RUN tar --strip-components=1 -xzvf goaccess.tar.gz -RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( GOAN<\/a> v1.1.24<\/span> )/" /goaccess/resources/tpls.html +RUN sed -i "s/GWSocket<\/a>/GWSocket<\/a> ( GOAN<\/a> v1.1.25<\/span> )/" /goaccess/resources/tpls.html RUN sed -i "s/bottom: 190px/bottom: 260px/" /goaccess/resources/css/app.css RUN ./configure --enable-utf8 --enable-geoip=mmdb --with-getline RUN make diff --git a/README.md b/README.md index baaa4b4..8d7fae9 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,9 @@ services: - SKIP_ARCHIVED_LOGS - by default the following file(s) are read and parsed. - *.log + - NGINX_ACCESS + - the following file(s) are read and parsed. + - access.log # **LOG FORMATS** @@ -207,6 +210,13 @@ date-format %d/%b/%Y log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" %Lm" ``` +### NGINX ACCESS LOG FORMAT +``` +time-format %T +date-format %d/%b/%Y +log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" +``` + # **Possible/Known Issues** - A lot of CPU Usage and 10000 request every second in webUI diff --git a/resources/scripts/logs/nginx_access.sh b/resources/scripts/logs/nginx_access.sh new file mode 100644 index 0000000..bc0a64e --- /dev/null +++ b/resources/scripts/logs/nginx_access.sh @@ -0,0 +1,121 @@ +#!/bin/bash +function nginx_access_init(){ + goan_config="/goaccess-config/goaccess.conf" + nginx_html="/var/www/html/index.html" + html_config="/var/www/html/goaccess_conf.html" + archive_log="/goaccess-config/archive.log" + active_log="/goaccess-config/active.log" + + if [[ -f ${goan_config} ]]; then + rm ${goan_config} + else + mkdir -p "/goaccess-config/" + cp /goaccess-config/goaccess.conf.bak ${goan_config} + fi + if [[ -f ${nginx_html} ]]; then + rm ${nginx_html} + else + mkdir -p "/var/www/html/" + touch ${nginx_html} + fi + if [[ -f ${html_config} ]]; then + rm ${html_config} + fi + + echo -n "" > ${archive_log} + echo -n "" > ${active_log} +} + +function nginx_access_goaccess_config(){ + echo -e "\n\n\n" >> ${goan_config} + echo "######################################" >> ${goan_config} + echo "# ${goan_version}" >> ${goan_config} + echo "# GOAN_PROXY_CONFIG" >> ${goan_config} + echo "######################################" >> ${goan_config} + echo "time-format %T" >> ${goan_config} + echo "date-format %d/%b/%Y" >> ${goan_config} + echo "log-format %h %^[%d:%t %^] \"%r\" %s %b \"%R\" \"%u\"" >> ${goan_config} + echo "port 7890" >> ${goan_config} + echo "real-time-html true" >> ${goan_config} + echo "output ${nginx_html}" >> ${goan_config} + if [[ "${ENABLE_BROWSERS_LIST}" == "True" || ${ENABLE_BROWSERS_LIST} == true ]]; then + echo -e "\n\tENABLING NGINX ACCESS INSTANCE GOACCESS BROWSERS LIST" + browsers_file="/goaccess-config/browsers.list" + echo "browsers-file ${browsers_file}" >> ${goan_config} + fi +} + +function nginx_access(){ + nginx_access_init + nginx_access_goaccess_config + + echo -e "\nLOADING NGINX ACCESS LOGS" + echo "-------------------------------" + + echo $'\n' >> ${goan_config} + echo "#GOAN_DEFAULT_LOG_FILES" >> ${goan_config} + echo "log-file ${archive_log}" >> ${goan_config} + echo "log-file ${active_log}" >> ${goan_config} + + goan_log_count=0 + goan_archive_log_count=0 + + echo -e "\n#GOAN_NGINX ACCESS_LOG_FILES" >> ${goan_config} + if [[ -d "${goan_log_path}" ]]; then + + echo -e "\n\tAdding proxy logs..." + IFS=$'\n' + + for file in $(find "${goan_log_path}" -name 'access.log' ! -name "error.log"); + do + if [ -f $file ] + then + if [ -r $file ] && R="Read = yes" || R="Read = No" + then + echo "log-file ${file}" >> ${goan_config} + goan_log_count=$((goan_log_count+1)) + echo -ne ' \t ' + echo "Filename: $file | $R" + else + echo -ne ' \t ' + echo "Filename: $file | $R" + fi + else + echo -ne ' \t ' + echo "Filename: $file | Not a file" + fi + done + unset IFS + else + echo "Problem loading directory (check directory or permissions)... ${goan_log_path}" + fi + + if [ $goan_log_count != 0 ] + then + echo "Found (${goan_log_count}) proxy logs..." + else + echo "No access.log found. Creating an empty log file..." + touch "${goan_log_path}/access.log" + fi + + #additonal config settings + exclude_ips ${goan_config} + debug ${goan_config} ${html_config} + set_geoip_database ${goan_config} + + echo -e "\nSKIP ARCHIVED LOGS" + echo "-------------------------------" + echo "FEATURE NOT AVAILABLE FOR NGINX ACCESS" + + #write out loading page + echo "" > ${nginx_html} + echo "GOAN - ${goan_version}" >> ${nginx_html} + echo "" >> ${nginx_html} + echo "" >> ${nginx_html} + echo "

${goan_version}

loading...

" >> ${nginx_html} + echo "Logs processing: $(($goan_log_count)) (might take some time depending on the number of files to parse)" >> ${nginx_html} + echo "

" >> ${nginx_html} + + echo -e "\nRUN NGINX ACCESS GOACCESS" + runGoAccess +} diff --git a/resources/scripts/start.sh b/resources/scripts/start.sh index 147c002..751edbb 100755 --- a/resources/scripts/start.sh +++ b/resources/scripts/start.sh @@ -7,8 +7,9 @@ source $(dirname "$0")/logs/npm_error.sh source $(dirname "$0")/logs/traefik.sh source $(dirname "$0")/logs/custom.sh source $(dirname "$0")/logs/ncsa_combined.sh +source $(dirname "$0")/logs/nginx_access.sh -goan_version="GOAN v1.1.24" +goan_version="GOAN v1.1.25" goan_log_path="/opt/log" goaccess_ping_interval=15 @@ -80,6 +81,8 @@ elif [[ "${LOG_TYPE}" == "NCSA_COMBINED" ]]; then ncsa_combined elif [[ "${LOG_TYPE}" == "CUSTOM" ]]; then custom +elif [[ "${LOG_TYPE}" == "NGINX_ACCESS" ]]; then + nginx_access fi # END PROXY LOGS From 7b9d7c3cf3b8f13c34b037c73fe17b9ffd497c2c Mon Sep 17 00:00:00 2001 From: Xavier Hernandez <6011223+xavier-hernandez@users.noreply.github.com> Date: Sun, 21 Jan 2024 22:36:40 -0600 Subject: [PATCH 2/2] v1.1.25 - fixed npm error issue with parsing log files, was display domain rather than file --- resources/scripts/logs/npm_error.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/scripts/logs/npm_error.sh b/resources/scripts/logs/npm_error.sh index ed04d86..762edb9 100644 --- a/resources/scripts/logs/npm_error.sh +++ b/resources/scripts/logs/npm_error.sh @@ -34,7 +34,7 @@ function npm_error_goaccess_config(){ echo "######################################" >> ${goan_config} echo "time-format %T" >> ${goan_config} echo "date-format %Y/%m/%d" >> ${goan_config} - echo "log_format %d %t %^: %v, %^: %h, %^ %v %^"%r" %^" >> ${goan_config} + echo "log_format %d %t %^: %v, %^: %h, %^ %v %^\"%r\" %^" >> ${goan_config} echo "port 7892" >> ${goan_config} echo "real-time-html true" >> ${goan_config} echo "output ${nginx_html}" >> ${goan_config}