Merging files

;;; Created by Matthias Müller
;;; Creative Commons License: This work is licensed under a Creative Commons Attribution License CCBY 4.0



extensions [csv table array]

to merge-files
  reset-timer
  let which-file user-file
  show (word "loading 1. file: " which-file)
  let input csv:from-file which-file
  if "[run number]" !=  first first input [
    repeat 6 [
      set input but-first input
    ]
  ]
  set input map [x -> but-first X] input
  let a 1
  repeat Number_of_files - 1 [
    set a a + 1
    set which-file user-file
    show (word "loading " a ". file: " which-file)
    let temp-input csv:from-file which-file
    ifelse "[run number]" !=  first first temp-input [
      repeat 7 [
        set temp-input but-first temp-input
      ]
    ][
      set temp-input but-first temp-input
    ]
    set temp-input map [x -> but-first X] temp-input
    ;; merge
    set input (sentence input temp-input)
  ]

  if delete-columns [
    show "deleting columns"
    let col-to-delete []
    foreach n-values length first input [?1 -> ?1] [col ->
      let temp length remove-duplicates map [row -> item col row] but-first input
      if temp = 1 [
        set col-to-delete lput col col-to-delete
      ]
    ]
    let temp-output []
    show (word length col-to-delete " unnecessary columns found")
    foreach input [row ->
      let temp-row row
      foreach reverse col-to-delete [col ->
        set temp-row remove-item col temp-row
      ]
      set temp-output lput temp-row temp-output
    ]
    set input temp-output
  ]

  csv:to-file output-file-merge-files input
  show (word "done in " timer " seconds")
  set input []
end 

to Merge-rows
  ca
  reset-timer
  let input csv:from-file user-file
  if file-has-netlogo-header? [
    repeat 6 [
      set input but-first input
    ]
    set input map [x -> but-first X] input
  ]
  let result []
  set result lput first input result
  ; find the ("[step]") column
  let sep 1 + position "[step]" first input
  ; get a list of uique parameter settings (para-sets)
  let para-sets remove-duplicates map [? -> sublist ? 0 sep ] but-first input
  show (word length para-sets " different parameter settings found")
  ;; copying rows of input into a temp. result file (temp-results) for each parameter setting
  let temp-results table:make
  foreach para-sets [? ->
    table:put temp-results ? []
  ]
  let counter 1
  foreach but-first input [rows ->
    if counter / 10000 = int (counter / 10000 )[
      if show-progress [ show (word "step: " counter ", " precision (counter / length input * 100) 2 "%")]
    ]
    set counter counter + 1
    ifelse table:get temp-results sublist rows 0 sep = [] [
      table:put temp-results sublist rows 0 sep (list sublist rows sep length rows)
    ][
      let temp table:get temp-results sublist rows 0 sep
      set temp lput ( sublist rows sep length rows) temp
      table:put temp-results (sublist rows 0 sep) temp
    ]
  ]
  set temp-results table:to-list temp-results
  show "merging"
  ;now go through the temp. results and merge the results for each parameter setting
  foreach temp-results [para-set ->
    let temp-mean []
    foreach n-values length first last para-set [x -> x] [pos ->
      set temp-mean lput mean map [elem -> item pos elem] last para-set temp-mean
    ]
    set result lput (sentence first para-set temp-mean) result
   ]
   show (word "done in: " timer " seconds")
   csv:to-file output-file-merge-rows result
end