# HG changeset patch
# User xitiomet
# Date 1251495440 14400
# Node ID ee2b834af76620f8936352d40a0a6bbeabc79598
# Parent  0000000000000000000000000000000000000000
Initial Commit for nginx log utility

diff -r 0000000000000000000000000000000000000000 -r ee2b834af76620f8936352d40a0a6bbeabc79598 Makefile
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Makefile	Fri Aug 28 17:37:20 2009 -0400
@@ -0,0 +1,12 @@
+CPC=g++
+
+all: nglog
+
+nglog: nglog.o
+	$(CPC) $< -o $@
+    
+%.o: %.cc
+	$(CPC) -I. -c $< -o $@
+
+clean:
+	rm -f nglog *~ *.o
diff -r 0000000000000000000000000000000000000000 -r ee2b834af76620f8936352d40a0a6bbeabc79598 nglog
Binary file nglog has changed
diff -r 0000000000000000000000000000000000000000 -r ee2b834af76620f8936352d40a0a6bbeabc79598 nglog.cc
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/nglog.cc	Fri Aug 28 17:37:20 2009 -0400
@@ -0,0 +1,298 @@
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include "vt100.h"
+
+using namespace std;
+
+struct Param
+{
+    string key;
+    string value;
+};
+
+struct Request
+{
+    string url;
+    vector<Param> params;
+};
+
+struct PreparedLog
+{
+    vector<Request> requests;
+    vector<string> columns;
+    float total_lines;
+};
+
+void unique_add(vector<string>& v, string& s)
+{
+    float i;
+    bool found = false;
+    for (i=0; i < v.size(); i++)
+    {
+        if (v.at(i) == s)
+        {
+            found = true;
+        }
+    }
+    if (!found)
+    {
+        v.push_back(s);
+    }
+}
+
+string get_param(vector<Param> v, string key)
+{
+    float i;
+    for (i=0; i < v.size(); i++)
+    {
+        Param e = v.at(i);
+        if (key == e.key)
+        {
+            return e.value;
+        }
+    }
+    return "";
+}
+
+PreparedLog prepareLog(istream& input_stream)
+{
+    string line;
+    float linenum = 0;
+    PreparedLog return_log;
+    while (getline (input_stream, line))
+    {
+        linenum++;
+
+        istringstream linestream(line);
+        string item;
+        int itemnum = 0;
+        while (getline (linestream, item, ' '))
+        {
+            itemnum++;
+            if (itemnum == 7)
+            {
+                // unique struct for line
+                Request this_request;
+
+                int qloc = item.find_first_of("?");
+                string url_params;
+                string url;
+
+                if (qloc != string::npos)
+                {
+                    this_request.url = item.substr(0, qloc);
+                    url_params = item.substr(qloc+1);
+
+                    int paramnumber = 0;
+                    istringstream paramstream(url_params);
+                    string param;
+
+                    // Check parameters to see what kind of colums we will have to create
+                    while (getline (paramstream, param, '&'))
+                    {
+                        int eqloc = param.find_first_of("=");
+                        if (eqloc != string::npos)
+                        {
+                            Param x;
+
+                            x.key = param.substr(0,eqloc);
+                            x.value = param.substr(eqloc+1);
+
+                            this_request.params.push_back(x);
+
+                            // if not in column db add it
+                            unique_add(return_log.columns, x.key);
+                        }
+                    }
+
+                } else {
+                    // No Parameters lets just get the url
+                    this_request.url = item;
+                }
+                return_log.requests.push_back(this_request);
+            }
+        }
+    }
+    return_log.total_lines = linenum;
+    return return_log;
+}
+
+void write_csv(PreparedLog& plog, ostream& outfile)
+{
+    outfile << "\"" << "url" << "\",";
+    float c;
+    for (c=0; c < plog.columns.size(); c++)
+    {
+        string f = plog.columns.at(c);
+        outfile << "\"" << f << "\"";
+        if (c+1 < plog.columns.size())
+        {
+            outfile << ",";
+        }
+    }
+    outfile << endl;
+
+    float i;
+    for (i=0; i < plog.requests.size(); i++)
+    {
+        Request n = plog.requests.at(i);
+        outfile << "\"" << n.url << "\",";
+        float b;
+        for (b=0; b < plog.columns.size(); b++)
+        {
+            string e = plog.columns.at(b);
+            outfile << "\"" << get_param(n.params, e) << "\"";
+            if (b+1 < plog.columns.size())
+            {
+                outfile << ",";
+            }
+        }
+        outfile << endl;
+    }
+}
+
+string view_vector(vector<string>& v)
+{
+    float i;
+    ostringstream oss;
+    for (i=0; i < v.size(); i++)
+    {
+        oss << v.at(i);
+        if (i+1 < v.size())
+        {
+            oss << ", ";
+        }
+    }
+    return oss.str();
+}
+
+
+int main (int argc, char *argv[])
+{
+    int i;
+    bool stdin = false;
+    bool stdout = false;
+    bool show_help = true;
+    bool input_set = false;
+    bool output_set = false;
+    bool quiet = false;
+    
+    string outFilename;
+    string inFilename;
+    
+    for (i=0; i<argc; i++)
+    {
+        string thisParam = argv[i];
+        if (!thisParam.compare("--help"))
+        {
+            show_help = true;
+        }
+        
+        if (!thisParam.compare("--pipe"))
+        {
+            stdin = true;
+            inFilename = "";
+            stdout = true;
+            inFilename = "";
+            
+            input_set = true;
+            output_set = true;
+        }
+        
+        if (!thisParam.compare("--quiet"))
+        {
+            quiet = true;
+        }
+        
+        if (!thisParam.compare("--stdin"))
+        {
+            stdin = true;
+            inFilename = "";
+            
+            input_set = true;
+        }
+        
+        if (!thisParam.compare("--stdout"))
+        {
+            stdout = true;
+            outFilename = "";
+            
+            output_set = true;
+        }
+
+        if (!thisParam.compare("-o"))
+        {
+            outFilename = argv[i+1];
+            stdout = false;
+            
+            output_set = true;
+        }
+
+        if (!thisParam.compare("-i"))
+        {
+            inFilename = argv[i+1];
+            stdin = false;
+            
+            input_set = true;
+        }
+    }
+    
+    if (input_set && output_set)
+    {
+        show_help = false;
+    }
+    
+    if (show_help)
+    {
+        cerr << "Usage: nglog [OPTION] -i [FILE] -o [FILE]" << endl;
+        cerr << "Process nginx logs for urls into csv files. Used to debug restfull api calls through nginx" << endl;
+        cerr << endl;
+        cerr << "Options:" << endl;
+        cerr << "  --pipe           Take input from stdin, and send output to stdout" << endl;
+        cerr << "  --stdin          Take input from stdin" << endl;
+        cerr << "  --stdout         Send output to stdout" << endl;
+        cerr << "  --quiet          No Messages" << endl;
+        cerr << "  -i               Specify Input file (NGINX LOG)" << endl;
+        cerr << "  -o               Specify Output file (CSV)" << endl << endl;
+        return 0;
+    }
+    
+    if (!quiet)
+    {
+        cerr << set_bold(true) << set_colors(VT_RED, VT_DEFAULT) << "NGINX Weblog Parser" << set_bold(false) << endl;
+        cerr << set_colors(VT_YELLOW, VT_DEFAULT)<< "Processing " << inFilename << "....";
+    }
+
+    istream *input_stream;
+    ifstream inFile;
+    if (!stdin)
+    {    
+        inFile.open(inFilename.c_str());
+        input_stream = &inFile;
+    } else {
+        input_stream = &cin;
+    }
+    PreparedLog plog = prepareLog(*input_stream);
+
+    if (!quiet)
+    {
+        cerr << set_colors(VT_DEFAULT, VT_DEFAULT) << "OK!" << set_colors(VT_DEFAULT, VT_DEFAULT) << endl;
+        cerr << set_colors(VT_YELLOW, VT_DEFAULT) << "Lines Processed: " << set_colors(VT_DEFAULT, VT_DEFAULT) << plog.total_lines << endl;
+        cerr << set_colors(VT_YELLOW, VT_DEFAULT) << "Columns Detected: " << set_colors(VT_DEFAULT, VT_DEFAULT) << view_vector(plog.columns) << endl;
+        cerr << finalize;
+    }
+    
+    if (!stdout)
+    {
+        ofstream outfile (outFilename.c_str());
+        write_csv(plog, outfile);
+        outfile.close();
+    } else {
+        write_csv(plog, cout);
+    }
+    return 0;
+}
+
diff -r 0000000000000000000000000000000000000000 -r ee2b834af76620f8936352d40a0a6bbeabc79598 nglog.o
Binary file nglog.o has changed
diff -r 0000000000000000000000000000000000000000 -r ee2b834af76620f8936352d40a0a6bbeabc79598 vt100.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/vt100.h	Fri Aug 28 17:37:20 2009 -0400
@@ -0,0 +1,202 @@
+// vt100.h
+/*
+ ----------------------------------------------
+ Terminal macros to use in C++ console programs
+ ----------------------------------------------
+
+ Use the following macros in your C++ cout statements to manipulate the
+ video display. Most systems emulate either VT100 terminals or the ANSI
+ terminal protocol. If yours does not, these won't work until you find
+ an ANSI terminal driver.
+
+ The following macros are practically guaranteed to work:
+   clear_screen
+   audible_bell
+   goto_xy()
+   move_to()
+   default_attributes
+   set_reverse()
+   set_colors()
+   finalize
+
+ The least likely to work are
+   show_cursor()
+   set_italic()
+
+ .............................................................................
+ Example usage:
+
+   cout << clear_screen
+        << goto_xy( 10, 2 ) << "Hello";
+
+ To terminate your program, your code should look like this:
+
+   cout << finalize;
+   return EXIT_SUCCESS;
+
+ .............................................................................
+ T H E   M A C R O S
+
+   cout << reset                       Restores the terminal to it's original
+                                       state.
+                                       You probably don't want to use this.
+
+   cout << clear_screen                Clear the entire screen.
+        << clear_to_bos                Clear from the top of the screen.
+        << clear_to_eos                Clear to the bottom of the screen.
+
+   cout << clear_to_eol                Clear to the end of the line.
+        << clear_to_bol                Clear to the beginning of the line.
+        << clear_line                  Clear the entire line.
+
+   cout << visual_bell                 Blinks the screen (unreliable).
+        << audible_bell                Sounds the speaker beep.
+
+   cout << show_cursor( TRUE )         Make the cursor visible (default)
+        << show_cursor( FALSE )        Make the cursor invisible
+
+   cout << goto_xy( x, y )             Move the cursor to coordinate (x, y),
+                                       or (column x, line y), where (1, 1)
+                                       is the upper-left corner of the screen.
+        << move_to( line, column )     Same.
+
+   cout << cursor_up(    count )       Move the cursor "count" characters
+        << cursor_down(  count )       (or lines) from its current position.
+        << cursor_left(  count )
+        << cursor_right( count )
+
+   cout << set_scrolling_region( top, bottom )
+                                       Set the lines in the range [top,bottom]
+                                       to be those that scroll.
+        << scroll_up(   count )        Scroll the region up or down "count"
+        << scroll_down( count )        lines.
+
+   cout << insert_line( count )        Insert or delete "count" items.
+        << delete_line( count )        Always inserts blanks.
+        << insert_char( count )        The result is affected if you change
+        << delete_char( count )        the scroll region.
+
+   cout << default_attributes          Reset so that the cursor attributes are
+                                       the default (no bold, underline, etc.
+                                       and colors are VT_DEFAULT).
+        << set_bold( bool )            Bold text.
+        << set_italic( bool )          Italic text.
+        << set_underline( bool )       Underline text.
+        << set_reverse( bool )         Reverse or invert the text colors.
+        << set_attributes( bold, italic, underline, reverse )
+                                       Set all four text attributes at once
+                                       without affecting the color attributes.
+        << set_colors( fg, bg )        The colors are
+                                         VT_BLACK    VT_YELLOW    VT_CYAN
+                                         VT_RED      VT_BLUE      VT_WHITE
+                                         VT_GREEN    VT_MAGENTA   VT_DEFAULT
+
+   cout << finalize                    Do this before your program ends.
+                                       (Alias for default_attributes.)
+
+ .............................................................................
+ Lastly
+
+   All this does is drive some basic functionality of a VT100/ANSI terminal.
+   Notibly, it lacks:
+    * The ability to tell you where the cursor is (so you must remember!)
+    * The ability to tell you the size of the terminal screen.
+      (You can use "stty size" at the command prompt to find out.)
+    * The ability to get just one character from the keyboard.
+      This takes some jumping through a few hoops. If you want to do some more
+      advanced stuff than what is presented here, consider using the ncurses
+      library (type "man ncurses" at the prompt).
+      (Here at RU, you'll have to include </usr/local/include/ncurses.h>
+      instead of just plain old <ncurses.h> to make curses work. Don't forget
+      to compile with "g++ ... -lncurses"!)
+   If you're interested in this kind of junk, type "man screen" at the prompt.
+
+ Enjoy!
+
+ --Michael Thomas Greer
+
+ */
+
+#ifndef VT100_MACRO_H
+#define VT100_MACRO_H
+
+#ifndef TRUE
+#define TRUE (1)
+#endif
+#ifndef FALSE
+#define FALSE (0)
+#endif
+
+#define VT_BLACK    0
+#define VT_RED      1
+#define VT_GREEN    2
+#define VT_YELLOW   3
+#define VT_BLUE     4
+#define VT_MAGENTA  5
+#define VT_CYAN     6
+#define VT_WHITE    7
+#define VT_DEFAULT  9
+
+#define reset "\33c"
+
+#define clear_screen "\33[2J"
+#define clear_to_bos "\33[1J"
+#define clear_to_eos "\33[J"
+
+#define clear_line   "\33[2K"
+#define clear_to_bol "\33[1K"
+#define clear_to_eol "\33[K"
+
+#define visual_bell  "\33g"
+#define audible_bell "\a"
+
+#define show_cursor( v ) ((v) ? "\33\67p" : "\33\66p")
+
+#define goto_xy( x, y ) "\33[" << y << ";" << x << "H"
+#define move_to( y, x ) "\33[" << y << ";" << x << "f"
+
+#define cursor_up(    count ) "\33[" << count << "A"
+#define cursor_down(  count ) "\33[" << count << "B"
+#define cursor_right( count ) "\33[" << count << "C"
+#define cursor_left(  count ) "\33[" << count << "D"
+
+#define set_scrolling_region( top, bottom ) "\33[" << top << ";" << bottom << "r"
+
+#define scroll_up(   count ) "\33[" << count << "S"
+#define scroll_down( count ) "\33[" << count << "T"
+
+#define insert_line( count ) "\33[" << count << "L"
+#define delete_line( count ) "\33[" << count << "M"
+
+#define insert_char( count ) "\33[" << count << "@"
+#define delete_char( count ) "\33[" << count << "P"
+
+static bool is_b = FALSE;
+static bool is_i = FALSE;
+static bool is_u = FALSE;
+static bool is_r = FALSE;
+static int  fg_c = VT_DEFAULT;
+static int  bg_c = VT_DEFAULT;
+
+#define default_attributes "\33[0m"
+
+// these four following are not meant to be used externally
+#define set_b( b ) ((is_b = b) ? "\33[1m" : "")
+#define set_i( i ) ((is_i = i) ? "\33[3m" : "")
+#define set_u( u ) ((is_u = u) ? "\33[4m" : "")
+#define set_r( r ) ((is_r = r) ? "\33[7m" : "")
+
+#define set_colors( fg, bg ) "\33[3" << (fg_c = fg) << ";4" << (bg_c = bg) << "m"
+
+#define set_attributes( b, i, u, r ) set_b( b ) << set_i( i ) << set_u( u ) << set_r( r ) << set_colors( fg_c, bg_c )
+
+#define set_bold(      b ) default_attributes << set_attributes( b, is_i, is_u, is_r )
+#define set_italic(    i ) default_attributes << set_attributes( is_b, i, is_u, is_r )
+#define set_underline( u ) default_attributes << set_attributes( is_b, is_i, u, is_r )
+#define set_reverse(   r ) default_attributes << set_attributes( is_b, is_i, is_u, r )
+
+#define finalize default_attributes
+
+#endif
+
+// end vt100.h