Merge pull request #200 from boyter/issue196

Issue196

Author: boyter

README.md

@@ -175,12 +175,12 @@ cost of running some static analysis tools.
 ### Usage
 
 Command line usage of `scc` is designed to be as simple as possible.
-Full details can be found in `scc --help` or `scc -h`.
+Full details can be found in `scc --help` or `scc -h`. Note that the below reflects the state of master not a release.
 
 ```
 $ scc -h
 Sloc, Cloc and Code. Count lines of code in a directory with complexity estimation.
-Version 2.13.0
+Version 3.0.0 (beta)
 Ben Boyter <[email protected]> + Contributors
 
 Usage:
@@ -195,7 +195,7 @@ Flags:
       --debug                       enable debug output
       --exclude-dir strings         directories to exclude (default [.git,.hg,.svn])
       --file-gc-count int           number of files to parse before turning the GC on (default 10000)
-  -f, --format string               set output format [tabular, wide, json, csv, cloc-yaml, html, html-table] (default "tabular")
+  -f, --format string               set output format [tabular, wide, json, csv, cloc-yaml, html, html-table, sql, sql-insert] (default "tabular")
       --format-multi string         have multiple format output overriding --format [e.g. tabular:stdout,csv:file.csv,json:file.json]
       --gen                         identify generated files
       --generated-markers strings   string markers in head of generated files (default [do not edit])
@@ -224,6 +224,7 @@ Flags:
       --remap-unknown string        inspect files of unknown type and remap by checking for a string and remapping the language [e.g. "-*- C++ -*-":"C Header"]
       --size-unit string            set size unit [si, binary, mixed, xkcd-kb, xkcd-kelly, xkcd-imaginary, xkcd-intel, xkcd-drive, xkcd-bakers] (default "si")
   -s, --sort string                 column to sort by [files, name, lines, blanks, code, comments, complexity] (default "files")
+      --sql-project string          use supplied name as the project identifier for the current run. Only valid with the --format sql or sql-insert option
   -t, --trace                       enable trace output (not recommended when processing multiple files)
   -v, --verbose                     verbose output
       --version                     version for scc
@@ -382,7 +383,7 @@ Note that in all cases if the remap rule does not apply normal #! rules will app
 
 By default `scc` will output to the console. However you can produce output in other formats if you require.
 
-The different options are `tabular, wide, json, csv, cloc-yaml, html, html-table`. 
+The different options are `tabular, wide, json, csv, cloc-yaml, html, html-table, sql, sql-insert`. 
 
 Note that you can write `scc` output to disk using the `-o, --output` option. This allows you to specify a file to
 write your output to. For example `scc -f html -o output.html` will run `scc` against the current directory, and output
@@ -491,6 +492,39 @@ file and not just the summary.
 Note that this format if it has the `--by-file` option will give you the byte size of every file it `scc` reads allowing you to get a breakdown of the
 number of bytes processed.
 
+#### SQL and SQL-Insert
+
+The SQL output format "mostly" compatible with cloc's SQL output format https://github.com/AlDanial/cloc#sql-
+
+While all queries on the cloc documentation should work as expected, you will not be able to append output from `scc` and `cloc` into the same database. This is because the table format is slightly different
+to account for scc including complexity counts and bytes.
+
+The difference between `sql` and `sql-insert` is that `sql` will include table creation while the latter will only have the insert commands.
+
+Usage is 100% the same as any other `scc` command but sql output will always contain per file details. You can compute totals yourself using SQL.
+
+The below will run scc against the current directory, name the ouput as the project scc and then pipe the output to sqlite to put into the database code.db
+
+```
+scc --format sql --sql-project scc . | sqlite3 code.db
+```
+
+Assuming you then wanted to append another project
+
+```
+scc --format sql-insert --sql-project redis . | sqlite3 code.db
+```
+
+You could then run SQL against the database,
+
+```
+sqlite3 code.db 'select project,file,max(nCode) as nL from t
+                         group by project order by nL desc;'
+```
+
+See the cloc documentation for more examples.
+
+
 ### Performance
 
 Generally `scc` will the fastest code counter compared to any I am aware of and have compared against. The below comparisons are taken from the fastest alternative counters. See `Other similar projects` above to see all of the other code counters compared against. It is designed to scale to as many CPU's cores as you can provide.

SCC-OUTPUT-REPORT.html

@@ -12,12 +12,12 @@
 	<tbody><tr>
 		<th>Go</th>
 		<th>34</th>
-		<th>8088</th>
-		<th>1323</th>
-		<th>347</th>
-		<th>6418</th>
-		<th>1338</th>
-		<th>321289</th>
+		<th>8208</th>
+		<th>1340</th>
+		<th>349</th>
+		<th>6519</th>
+		<th>1352</th>
+		<th>324777</th>
 	</tr><tr>
 		<th>Java</th>
 		<th>24</th>
@@ -48,12 +48,12 @@
 	</tr><tr>
 		<th>Markdown</th>
 		<th>8</th>
-		<th>1133</th>
-		<th>265</th>
+		<th>1189</th>
+		<th>283</th>
 		<th>0</th>
-		<th>868</th>
+		<th>906</th>
 		<th>0</th>
-		<th>46552</th>
+		<th>48490</th>
 	</tr><tr>
 		<th>CSS</th>
 		<th>5</th>
@@ -93,12 +93,12 @@
 	</tr><tr>
 		<th>Shell</th>
 		<th>3</th>
-		<th>950</th>
-		<th>132</th>
+		<th>987</th>
+		<th>136</th>
 		<th>37</th>
-		<th>781</th>
-		<th>87</th>
-		<th>33879</th>
+		<th>814</th>
+		<th>91</th>
+		<th>35251</th>
 	</tr><tr>
 		<th>JavaServer Pages</th>
 		<th>2</th>
@@ -553,11 +553,11 @@
 	<tfoot><tr>
 		<th>Total</th>
 		<th>162</th>
-		<th>24483</th>
-		<th>2791</th>
-		<th>1540</th>
-		<th>20152</th>
-		<th>2258</th>
-    	<th>1738364</th>
+		<th>24696</th>
+		<th>2830</th>
+		<th>1542</th>
+		<th>20324</th>
+		<th>2276</th>
+    	<th>1745162</th>
 	</tr></tfoot>
 	</table></body></html>

main.go

@@ -93,7 +93,7 @@ func main() {
 		"format",
 		"f",
 		"tabular",
-		"set output format [tabular, wide, json, csv, cloc-yaml, html, html-table]",
+		"set output format [tabular, wide, json, csv, cloc-yaml, html, html-table, sql, sql-insert]",
 	)
 	flags.StringSliceVarP(
 		&processor.AllowListExtensions,
@@ -271,6 +271,12 @@ func main() {
 		"",
 		"have multiple format output overriding --format [e.g. tabular:stdout,csv:file.csv,json:file.json]",
 	)
+	flags.StringVar(
+		&processor.SQLProject,
+		"sql-project",
+		"",
+		"use supplied name as the project identifier for the current run. Only valid with the --format sql or sql-insert option",
+	)
 	flags.StringVar(
 		&processor.RemapUnknown,
 		"remap-unknown",

processor/formatters.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -411,6 +412,66 @@ func toHtmlTable(input chan *FileJob) string {
 	return str.String()
 }
 
+func toSqlInsert(input chan *FileJob) string {
+	var str strings.Builder
+	projectName := SQLProject
+	if projectName == "" {
+		projectName = strings.Join(DirFilePaths, ",")
+	}
+
+	str.WriteString("\nbegin transaction;")
+	count := 0
+	for res := range input {
+		count++
+
+		dir, _ := filepath.Split(res.Location)
+
+		str.WriteString(fmt.Sprintf("\ninsert into t values('%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, %d);",
+			projectName, res.Language, res.Location, dir, res.Filename, res.Bytes, res.Blank, res.Comment, res.Code, res.Complexity))
+
+		// every 1000 files commit and start a new transaction to avoid overloading
+		if count == 1000 {
+			str.WriteString("\ncommit;")
+			str.WriteString("\nbegin transaction;")
+			count = 0
+		}
+	}
+	if count != 1000 {
+		str.WriteString("\ncommit;")
+	}
+
+	currentTime := time.Now()
+	es := float64(makeTimestampMilli()-startTimeMilli) * 0.001
+	str.WriteString("\nbegin transaction;")
+	str.WriteString(fmt.Sprintf("\ninsert into metadata values('%s', '%s', %f);", currentTime.Format("2006-01-02 15:04:05"), projectName, es))
+	str.WriteString("\ncommit;")
+
+	return str.String()
+}
+
+func toSql(input chan *FileJob) string {
+	var str strings.Builder
+
+	str.WriteString(`create table metadata (   -- github.com/boyter/scc v ` + Version + `
+             timestamp text,
+             Project   text,
+             elapsed_s real);
+create table t        (
+             Project       text   ,
+             Language      text   ,
+             File          text   ,
+             File_dirname  text   ,
+             File_basename text   ,
+             nByte         integer,
+             nBlank        integer,
+             nComment      integer,
+             nCode         integer,
+             nComplexity   integer   );`)
+
+	str.WriteString(toSqlInsert(input))
+	return str.String()
+}
+
 func fileSummarize(input chan *FileJob) string {
 	if FormatMulti != "" {
 		return fileSummarizeMulti(input)
@@ -429,6 +490,10 @@ func fileSummarize(input chan *FileJob) string {
 		return toHtml(input)
 	case strings.ToLower(Format) == "html-table":
 		return toHtmlTable(input)
+	case strings.ToLower(Format) == "sql":
+		return toSql(input)
+	case strings.ToLower(Format) == "sql-insert":
+		return toSqlInsert(input)
 	}
 
 	return fileSummarizeShort(input)
@@ -476,6 +541,10 @@ func fileSummarizeMulti(input chan *FileJob) string {
 				val = toHtml(i)
 			case "html-table":
 				val = toHtmlTable(i)
+			case "sql":
+				val = toSql(i)
+			case "sql-insert":
+				val = toSqlInsert(i)
 			}
 
 			if t[1] == "stdout" {

processor/formatters_test.go

@@ -606,6 +606,48 @@ func TestToCsvMultiple(t *testing.T) {
 	}
 }
 
+func TestToSQLSingle(t *testing.T) {
+	inputChan := make(chan *FileJob, 1000)
+	inputChan <- &FileJob{
+		Language:           "Go",
+		Filename:           "bbbb.go",
+		Extension:          "go",
+		Location:           "./",
+		Bytes:              1000,
+		Lines:              1000,
+		Code:               1000,
+		Comment:            1000,
+		Blank:              1000,
+		Complexity:         1000,
+		WeightedComplexity: 1000,
+		Binary:             false,
+	}
+	close(inputChan)
+	Debug = true // Increase coverage slightly
+	res := toSql(inputChan)
+	Debug = false
+
+	if !strings.Contains(res, `create table metadata`) {
+		t.Error("Expected create table return", res)
+	}
+
+	if !strings.Contains(res, `create table t`) {
+		t.Error("Expected create table return", res)
+	}
+
+	if !strings.Contains(res, `begin transaction`) {
+		t.Error("Expected begin transaction return", res)
+	}
+
+	if !strings.Contains(res, `insert into t values('', 'Go', './', './', 'bbbb.go', 1000, 1000, 1000, 1000, 1000);`) {
+		t.Error("Expected insert return", res)
+	}
+
+	if !strings.Contains(res, `insert into metadata values`) {
+		t.Error("Expected insert return", res)
+	}
+}
+
 func TestFileSummarizeWide(t *testing.T) {
 	inputChan := make(chan *FileJob, 1000)
 	inputChan <- &FileJob{

processor/processor.go

@@ -15,7 +15,7 @@ import (
 )
 
 // Version indicates the version of the application
-var Version = "2.13.0"
+var Version = "3.0.0 (beta)"
 
 // Flags set via the CLI which control how the output is displayed
 
@@ -103,6 +103,9 @@ var Format = ""
 // FormatMulti is a rule for defining multiple output formats
 var FormatMulti = ""
 
+// SQLProject is used to store the name for the SQL insert formats but is optional
+var SQLProject = ""
+
 // RemapUnknown allows remapping of unknown files with a string to search the content for
 var RemapUnknown = ""