Brendan Tompkins [MVP]

I think the post title is clear enough.   Once you have a Typed DataSet, you can do anything, in my opinion.   Recently, we were having a problem analyzing our IIS logs with WebTrends.  The webmaster here asked me if I could write something in .NET that would let us quickly analyze our logs.  We have this tool that's basically a fancy DataGrid with some PivotTable and PivotChart capabilites sprinkled on top, and it turns out to be perfect for analyzing this kind of data.  All I needed to use this was a Typed DataSet.  So, after a bit of coding, I came up with the following Typed DataSet and Utility Code to fill it from IIS log files.  I've used this to analyze a weeks worth of 1M logs by merging each DataSet before binding it to our grid. This chugs a bit, but works. 

DataSet:

<xs:element name="IISLogAnalysis" msdata:IsDataSet="true">
  <xs:complexType>
    <xs:choice maxOccurs="unbounded">
    <xs:element name="IISLog">
    <xs:complexType>
      <xs:sequence>
        <xs:element name="Log_x0020_Date" minOccurs="0" type="xs:dateTime" />
        <xs:element name="Client_x0020_IP" type="xs:string" minOccurs="0" />
        <xs:element name="Method" type="xs:string" minOccurs="0" />
        <xs:element name="URL" type="xs:string" minOccurs="0" />
        <xs:element name="Query_x0020_String" type="xs:string" minOccurs="0" />
        <xs:element name="Status_x0020_Code" type="xs:string" minOccurs="0" />
        <xs:element name="User_x0020_Agent" type="xs:string" minOccurs="0" /> 
      </xs:sequence>
    </xs:complexType>
    </xs:element>
    </xs:choice>
  </xs:complexType>
</xs:element>

Static methods for log analysis.

public static IISLogAnalysis AnalyizeLogFile(string logFileName, string [] strExcludeMethods, string [] strExcludeStatusCodes, string [] strExcludeFileExts)
{
   StreamReader sr = new StreamReader(logFileName);
   IISLogAnalysis anal = new IISLogAnalysis();
   int timePos = -1;
   int ipPos = -1;
   int methodPos = -1;
   int pagePos = -1;
   int queryStringPos = -1;
   int statusPos = -1;
   int userAgentPos = -1;

   System.DateTime logDate = new System.DateTime();   
   
   while (sr.Peek() >= 0) 
   {
      string fileLine = sr.ReadLine();

      // All logs have time
      if(timePos == -1)
      {
            if(fileLine.StartsWith("#Fields:"))
            {
                  // Get our field positions
                  string revisedColmNames=fileLine.Replace("#Fields: ","");
                  string[] arColm=revisedColmNames.Split(Convert.ToChar(" "));
                  ArrayList al= new ArrayList(arColm);
                  timePos = al.IndexOf("time");
                  ipPos = al.IndexOf("c-ip");
                  methodPos = al.IndexOf("cs-method");
                  pagePos = al.IndexOf("cs-uri-stem");
                  queryStringPos = al.IndexOf("cs-uri-query");
                  statusPos = al.IndexOf("sc-status");
                  userAgentPos = al.IndexOf("cs(User-Agent)");
             }
             else if(fileLine.StartsWith("#Date:"))
             {
                  // Get our Date
                  logDate = System.DateTime.Parse(fileLine.Replace("#Date: ", ""));
             }
      }
      else
      {
            if(!fileLine.StartsWith("#") && fileLine.Trim().Length > 0)
            {
                  try
                  {
                        string []logEntry = fileLine.Split(new char[] {' '});
                        DateTime entryDate = System.DateTime.Parse(logDate.ToShortDateString() + " " 
                                                                + logEntry[timePos]);
                        string strIpAddress = (ipPos > -1) ? logEntry[ipPos]: null;
                        string strMethod = (methodPos > -1) ? logEntry[methodPos]: null;
                        string strURL = (pagePos > -1) ? logEntry[pagePos]: null;
                        string strQueryString = (queryStringPos > -1) ? logEntry[queryStringPos]: null;
                        string strStatus = (statusPos > -1) ? logEntry[statusPos]: null;
                        string strUserAgent = (userAgentPos > -1) ? logEntry[userAgentPos]: null;
                 
                        // Only log the entry if we are not excluding it
                        bool doLog = !(strExcludeMethods != null && 
                                                IsStringInArray(strMethod, strExcludeMethods, true));
                        if(doLog && strExcludeFileExts != null)
                        {
                                string fileExt = new FileInfo(strURL).Extension;
                                doLog = !(IsStringInArray(fileExt, strExcludeFileExts, true));
                        }
                        if(doLog) 
                                doLog = !(strExcludeStatusCodes != null && 
                                                IsStringInArray(strStatus, strExcludeStatusCodes, true));
                        if(doLog) 
                               anal.IISLog.AddIISLogRow(entryDate, strIpAddress, strMethod, strURL, 
                                                strQueryString, strStatus, strUserAgent);
                    }
                    catch(System.IndexOutOfRangeException){}
                  }
            }
       }

       sr.Close();
       return anal;
}

public static bool IsStringInArray(string strSearch, string [] strArray, bool ignoreCase)
{
   foreach(string s in strArray)
   {
      if(ignoreCase && (s.ToUpper() == strSearch.ToUpper())) return true;
      else if(s == strSearch) return true;
    }
   
    return false;
}