Differences

This shows you the differences between two versions of the page.

--- log_hits [2016/05/20 07:01] – luke7858
+++ log_hits [2024/05/23 07:26] (current) – external edit 127.0.0.1
@@ Line 12: / Line 12: @@
 for log in $(echo "$LOGS" | grep access); do HIT_COUNT=$(grep $(date "+%d/%b/%Y") $log -c); if [[ "$HIT_COUNT" -ge 100 ]]; then echo -e "\n$log - $HIT_COUNT total hits today\n"; grep $(date "+%d/%b/%Y") $log | awk -F \" '{USER[$(NF-1)]++}END{for (i in USER) print USER[i],i}' | sort -n | tail -10 ; fi; done
 </sxh>
-\\
 \\
 Example Output:
@@ Line 52: / Line 50: @@
 113.190.128.197 "POST /xmlrpc.php
 117.4.251.108 "POST /xmlrpc.php
+</sxh>
+\\
+=== Finding Crawlers ===
+**Note:** You will need to change date range and make sure you run the very first LOG command on this page first.
+<sxh bash>
+LC_ALL=C awk '/21\/Jul\/2016:20:4/ {REQ[FILENAME" "substr($0,index($0,$12))]++}END{for (i in REQ) print REQ[i],i}' $(echo "$LOGS" | grep access) | sort -rn | egrep -i "bot|crawl|spider|slurp" | head -25
+</sxh>
+\\
+=== Accurate number of Apache requests per hour ===
+**Note:** Change date range and log file locationg
+<sxh bash>
+LC_ALL=C awk '/02\/Aug\/2016/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP|.*Monitoring/' /var/log/nginx/exampledomain.com | cut -d[ -f2 | cut -d] -f1 | awk -F: '{print $2":00"}' | sort -n | uniq -c
+</sxh>
+\\
+=== Log requests for Specific date/time range ===
+**Note:** Change date/time and log file location
+<sxh bash>
+LC_ALL=C awk -F \" '/09\/Jun\/2016:(10:[12345]|11:[01])/ && $0 !~ /(.js|.png|.jpg|.css|.ico) HTTP/ {REQ[$2]++}END{for (i in REQ) print REQ[i],i}' /var/log/httpd/lexampledomain.com | sort -rn | head -50
 </sxh>