nglog / nglog.cc

#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <string>
#include "vt100.h"

using namespace std;

struct Param
{
    string key;
    string value;
};

struct Request
{
    string url;
    string timestamp;
    vector<Param> params;
};

struct PreparedLog
{
    vector<Request> requests;
    vector<string> columns;
    float total_lines;
};

void unique_add(vector<string>& v, string& s)
{
    unsigned int i;
    bool found = false;
    for (i=0; i < v.size(); i++)
    {
        if (v.at(i) == s)
        {
            found = true;
        }
    }
    if (!found)
    {
        v.push_back(s);
    }
}

string get_param(vector<Param> v, string key)
{
    unsigned int i;
    for (i=0; i < v.size(); i++)
    {
        Param e = v.at(i);
        if (key == e.key)
        {
            return e.value;
        }
    }
    return "";
}

PreparedLog prepareLog(istream& input_stream)
{
    string line;
    unsigned int linenum = 0;
    PreparedLog return_log;
    while (getline (input_stream, line))
    {
        linenum++;
        // unique struct for line
        Request this_request;
        istringstream linestream(line);
        string item;
        int itemnum = 0;
        while (getline (linestream, item, ' '))
        {
            itemnum++;
            if (itemnum == 4)
            {
               this_request.timestamp = item.substr(1);
            } 
            if (itemnum == 7)
            {
                int qloc = item.find_first_of("?");
                string url_params;
                string url;

                if (qloc != string::npos)
                {
                    this_request.url = item.substr(0, qloc);
                    url_params = item.substr(qloc+1);

                    int paramnumber = 0;
                    istringstream paramstream(url_params);
                    string param;

                    // Check parameters to see what kind of colums we will have to create
                    while (getline (paramstream, param, '&'))
                    {
                        int eqloc = param.find_first_of("=");
                        if (eqloc != string::npos)
                        {
                            Param x;

                            x.key = param.substr(0,eqloc);
                            x.value = param.substr(eqloc+1);

                            this_request.params.push_back(x);

                            // if not in column db add it
                            unique_add(return_log.columns, x.key);
                        }
                    }

                } else {
                    // No Parameters lets just get the url
                    this_request.url = item;
                }
                return_log.requests.push_back(this_request);
            }
        }
    }
    return_log.total_lines = linenum;
    return return_log;
}

void write_csv(PreparedLog& plog, ostream& outfile)
{
    outfile << "\"timestamp\",\"url\",";
    unsigned int c;
    for (c=0; c < plog.columns.size(); c++)
    {
        string f = plog.columns.at(c);
        outfile << "\"" << f << "\"";
        if (c+1 < plog.columns.size())
        {
            outfile << ",";
        }
    }
    outfile << endl;

    unsigned int i;
    for (i=0; i < plog.requests.size(); i++)
    {
        Request n = plog.requests.at(i);
        outfile << "\"" << n.timestamp << "\"," << "\"" << n.url << "\",";
        unsigned int b;
        for (b=0; b < plog.columns.size(); b++)
        {
            string e = plog.columns.at(b);
            outfile << "\"" << get_param(n.params, e) << "\"";
            if (b+1 < plog.columns.size())
            {
                outfile << ",";
            }
        }
        outfile << endl;
    }
}

string view_vector(vector<string>& v)
{
    unsigned int i;
    ostringstream oss;
    for (i=0; i < v.size(); i++)
    {
        oss << v.at(i);
        if (i+1 < v.size())
        {
            oss << ", ";
        }
    }
    return oss.str();
}


int main (int argc, char *argv[])
{
    int i;
    bool std_in = false;
    bool std_out = false;
    bool show_help = true;
    bool input_set = false;
    bool output_set = false;
    bool quiet = false;
    
    string outFilename;
    string inFilename;
    
    for (i=0; i<argc; i++)
    {
        string thisParam = argv[i];
        if (!thisParam.compare("--help"))
        {
            show_help = true;
        }
        
        if (!thisParam.compare("--pipe"))
        {
            std_in = true;
            inFilename = "";
            std_out = true;
            inFilename = "";
            
            input_set = true;
            output_set = true;
        }
        
        if (!thisParam.compare("--quiet"))
        {
            quiet = true;
        }
        
        if (!thisParam.compare("--stdin"))
        {
            std_in = true;
            inFilename = "";
            
            input_set = true;
        }
        
        if (!thisParam.compare("--stdout"))
        {
            std_out = true;
            outFilename = "";
            
            output_set = true;
        }

        if (!thisParam.compare("-o"))
        {
            outFilename = argv[i+1];
            std_out = false;
            
            output_set = true;
        }

        if (!thisParam.compare("-i"))
        {
            inFilename = argv[i+1];
            std_in = false;
            
            input_set = true;
        }
    }
    
    if (input_set && output_set)
    {
        show_help = false;
    }
    
    if (show_help)
    {
        cerr << "Usage: nglog [OPTION] -i [FILE] -o [FILE]" << endl;
        cerr << "Process nginx logs for urls into csv files. Used to debug restfull api calls through nginx" << endl;
        cerr << endl;
        cerr << "Options:" << endl;
        cerr << "  --pipe           Take input from stdin, and send output to stdout" << endl;
        cerr << "  --stdin          Take input from stdin" << endl;
        cerr << "  --stdout         Send output to stdout" << endl;
        cerr << "  --quiet          No Messages" << endl;
        cerr << "  -i               Specify Input file (NGINX LOG)" << endl;
        cerr << "  -o               Specify Output file (CSV)" << endl << endl;
        return 0;
    }
    
    if (!quiet)
    {
        cerr << set_bold(true) << set_colors(VT_RED, VT_DEFAULT) << "NGINX Weblog Parser" << set_bold(false) << endl;
        cerr << set_colors(VT_YELLOW, VT_DEFAULT)<< "Processing " << inFilename << "....";
    }

    istream *input_stream;
    ifstream inFile;
    if (!std_in)
    {    
        inFile.open(inFilename.c_str());
        input_stream = &inFile;
    } else {
        input_stream = &cin;
    }
    PreparedLog plog = prepareLog(*input_stream);

    if (!quiet)
    {
        cerr << set_colors(VT_DEFAULT, VT_DEFAULT) << "OK!" << set_colors(VT_DEFAULT, VT_DEFAULT) << endl;
        cerr << set_colors(VT_YELLOW, VT_DEFAULT) << "Lines Processed: " << set_colors(VT_DEFAULT, VT_DEFAULT) << plog.total_lines << endl;
        cerr << set_colors(VT_YELLOW, VT_DEFAULT) << "Columns Detected: " << set_colors(VT_DEFAULT, VT_DEFAULT) << view_vector(plog.columns) << endl;
        cerr << finalize;
    }
    
    if (!std_out)
    {
        ofstream outfile (outFilename.c_str());
        write_csv(plog, outfile);
        outfile.close();
    } else {
        write_csv(plog, cout);
    }
    return 0;
}