Neural Network Model of Addiction

;; Spiking Neural Networks with STDP learning

;; Author: Cristian Jimenez Romero - The Open University - 2015


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Spiking Neural Network related code:;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
breed [neurontypes neurontype]
breed [inputneurons inputneuron]
breed [normalneurons normalneuron]
directed-link-breed [normal-synapses normal-synapse]
directed-link-breed [input-synapses input-synapse]

neurontypes-own
[

  neurontypeid
  ;;;;;;;;;;Neuron Dynamics;;;;;;;;;;;
  restingpotential
  firethreshold
  decayrate ;Decay rate constant
  relrefractorypotential ;Refractory potential
  intervrefractoryperiod ;Duration of absolute-refractory period
  minmembranepotential ;lowest boundary for membrane potential
  ;;;;;;;Learning Parameters;;;;;;;;;
  pos_hebb_weight ;;Weight to increase the efficacy of synapses
  pos_time_window ;; Positive learning window
  neg_hebb_weight ;;Weight to decrease the efficacy of synapses
  neg_time_window ;; negative learning window
  max_synaptic_weight ;;Maximum synaptic weight
  min_synaptic_weight ;;Minimum synaptic weight
  pos_syn_change_interval ;;ranges of pre–to–post synaptic interspike intervals over which synaptic change occur.
  neg_syn_change_interval

]

;;;
;;; Create a neuron type
;;; Parameters: Type-identifier, resting potential, firing threshold, decay rate, refractory potential, duration of refractory period, lowest boundary for membrane potential

to setup-neurontype [#pneurontypeid #prestpot #pthreshold #pdecayr #prefractpot #pintrefrectp #minmembranepotential]

  create-neurontypes 1
  [
     set shape "square"
     set neurontypeid #pneurontypeid
     set restingpotential #prestpot
     set firethreshold #pthreshold
     set decayrate #pdecayr
     set relrefractorypotential #prefractpot
     set intervrefractoryperiod #pintrefrectp
     set minmembranepotential #minmembranepotential
  ]
end 


;;;
;;; Set learning parameters for neuron type pneurontypeid
;;;

to set-neurontype-learning-params [ #pneurontypeid #ppos_hebb_weight #ppos_time_window #pneg_hebb_weight #pneg_time_window #pmax_synaptic_weight
                                    #pmin_synaptic_weight #ppos_syn_change_interval #pneg_syn_change_interval]

  ask neurontypes with [neurontypeid = #pneurontypeid]
  [
    set pos_hebb_weight #ppos_hebb_weight
    set pos_time_window #ppos_time_window
    set neg_hebb_weight #pneg_hebb_weight
    set neg_time_window #pneg_time_window
    set max_synaptic_weight #pmax_synaptic_weight
    set min_synaptic_weight #pmin_synaptic_weight
    set pos_syn_change_interval #ppos_syn_change_interval
    set neg_syn_change_interval #pneg_syn_change_interval
  ]
end 

;;;
;;; Declare an existing neuron "pneuronid" as neuron-type "pneurontypeid"
;;;

to set-neuron-to-neurontype [ #pneurontypeid #pneuronid ] ;Call by observer


    ask neurontypes with [neurontypeid = #pneurontypeid]
    [
      ask normalneuron #pneuronid
      [
        set nrestingpotential [restingpotential] of myself
        set nmembranepotential [restingpotential] of myself
        set nfirethreshold [firethreshold] of myself
        set ndecayrate [decayrate] of myself
        set nrelrefractorypotential [relrefractorypotential] of myself
        set nintervrefractoryperiod [intervrefractoryperiod] of myself
        set nminmembranepotential [minmembranepotential] of myself

        ;;;;;;;;;;;;;;;;Learning Parameters;;;;;;;;;;;;;;;;;
        set npos_hebb_weight [pos_hebb_weight] of myself
        set npos_time_window [pos_time_window] of myself
        set nneg_hebb_weight [neg_hebb_weight] of myself
        set nneg_time_window [neg_time_window] of myself
        set nmax_synaptic_weight [max_synaptic_weight] of myself
        set nmin_synaptic_weight [min_synaptic_weight] of myself
        set npos_syn_change_interval [pos_syn_change_interval] of myself
        set nneg_syn_change_interval [neg_syn_change_interval] of myself
      ]
     ]
end 


normalneurons-own [

  nlayernum
  nneuronid
  nneuronstate
  nrestingpotential
  nfirethreshold
  nmembranepotential
  nrelrefractorypotential
  nintervrefractoryperiod
  nrefractorycounter
  naxondelay
  nsynapsesarray
  nnumofsynapses
  nlast-firing-time
  nincomingspikes
  nlastspikeinput
  ndecayrate
  nneuronlabel
  nminmembranepotential
  ;;;;;;;;;;;;;;;;Learning Parameters;;;;;;;;;;;;;;;;;
  npos_hebb_weight ;;Weight to increase the efficacy of synapses
  npos_time_window ;; Positive learning window
  nneg_hebb_weight ;;Weight to decrease the efficacy of synapses
  nneg_time_window ;; negative learning window
  nmax_synaptic_weight ;;Maximum synaptic weight
  nmin_synaptic_weight ;;Minimum synaptic weight
  npos_syn_change_interval ;;ranges of pre–to–post synaptic interspike intervals over which synaptic change occur.
  nneg_syn_change_interval

]


normal-synapses-own [

  presynneuronlabel
  possynneuronlabel
  presynneuronid
  possynneuronid
  synapseefficacy
  exc_or_inh
  synapsedelay
  joblist
  learningon?

]


inputneurons-own [

   layernum
   neuronid
   neuronstate
   pulsecounter ;;Count the number of sent pulses
   interspikecounter ;;Count the time between pulses
   numberofspikes ;;Number of spikes to send
   postsynneuron
   encodedvalue
   isynapseefficacy ;;In most cases should be large enough to activate possynn with a single spike
   neuronlabel

]

to-report get-input-neuronid-from-label [#pneuronlabel]

  let returned_id nobody
  ask one-of inputneurons with [neuronlabel = #pneuronlabel][set returned_id neuronid]
  report returned_id
end 

to-report get-neuronid-from-label [#pneuronlabel]

  let returned_id nobody
  ask one-of normalneurons with [nneuronlabel = #pneuronlabel][set returned_id nneuronid]
  report returned_id
end 


;;;
;;; Create a new synapse between pre-synaptic neuron: #ppresynneuronlabel and post-synaptic neuron: #ppossynneuronlabel
;;;

to setup-synapse [#ppresynneuronlabel #ppossynneuronlabel #psynapseefficacy #pexc_or_inh #psyndelay #plearningon?]

   let presynneuid get-neuronid-from-label #ppresynneuronlabel
   let possynneuid get-neuronid-from-label #ppossynneuronlabel

   let postsynneu normalneuron possynneuid
   ask normalneuron presynneuid [
        create-normal-synapse-to postsynneu [
           set presynneuronlabel #ppresynneuronlabel
           set possynneuronlabel #ppossynneuronlabel
           set presynneuronid presynneuid
           set possynneuronid possynneuid
           set synapseefficacy #psynapseefficacy
           set exc_or_inh #pexc_or_inh
           set synapsedelay #psyndelay
           set joblist []
           set learningon? #plearningon?
           ifelse (#pexc_or_inh = inhibitory_synapse)
           [
             set color red
           ]
           [
             set color grey
           ]
           set thickness 0.5
        ]
     ]
end 


;;;
;;; Process incoming pulse from input neuron
;;;

to receive-input-neuron-pulse [ #psnefficacy #pexcinh ];;called by Neuron

  if ( nneuronstate != neuron_state_refractory )
  [
      ;;Adjust membrane potential:
      ifelse ( #pexcinh = excitatory_synapse )
      [
          set nmembranepotential nmembranepotential + #psnefficacy  ;;increase membrane potential
      ]
      [
          set nmembranepotential nmembranepotential - #psnefficacy ;;decrease membrane potential
          if (nmembranepotential < nminmembranepotential) ;; Floor for the membrane potential in case of extreme inhibition
          [
             set nmembranepotential nminmembranepotential
          ]
      ]
  ]
  set nlastspikeinput ticks
end 


;;;
;;; Neuron pstneuronid# processes incoming pulse from neuron #prneuronid
;;;

to receive-pulse [ #prneuronid #snefficacy #excinh #plearningon? ];;called by neuron

  if ( nneuronstate != neuron_state_refractory )
  [
      ;;Perturb membrane potential:
      ifelse ( #excinh = excitatory_synapse )
      [
          set nmembranepotential nmembranepotential + (#snefficacy * epsp-kernel) ;;increase membrane potential
      ]
      [
          set nmembranepotential nmembranepotential - (#snefficacy * ipsp-kernel) ;;decrease membrane potential
          if (nmembranepotential < nminmembranepotential) ;; Floor for the membrane potential in case of extreme inhibition
          [
             set nmembranepotential nminmembranepotential
          ]
      ]
  ]
  ;;Remember last input spike:
  set nlastspikeinput ticks
  ;; If plasticity is activated then store pulse info for further processing and apply STDP
  if (#plearningon? and learningmode?)
  [
     ;;Don't let incoming-pulses history-buffer grow beyond limits (delete oldest spike):
     check-pulse-history-buffer
     ;;Create list of parameters and populate it:
     let pulseinflist[]
     set pulseinflist lput #prneuronid pulseinflist ;;Add Presynaptic neuron Id
     set pulseinflist lput #snefficacy pulseinflist ;;Add Synaptic efficacy
     set pulseinflist lput #excinh pulseinflist ;;Add excitatory or inhibitory info.
     set pulseinflist lput ticks pulseinflist ;;Add arriving time
     set pulseinflist lput false pulseinflist ;;Indicate if pulse has been processed as an EPSP following a Postsynaptic spike ( Post -> Pre, negative hebbian)
     ;;Add list of parameters to list of incoming pulses:
     set nincomingspikes lput pulseinflist nincomingspikes
     ;;Apply STDP learning rule:
     apply-stdp-learning-rule
  ]
end 


;;;
;;; Neuron fires
;;;

to prepare-pulse-firing ;;Called by Neurons

   ;;Remember last firing time
   set nlast-firing-time ticks
   ;;Apply learning rule and after that empty incoming-pulses history:
   apply-stdp-learning-rule
   empty-pulse-history-buffer
   ;;transmit Pulse to postsynaptic neurons:
   propagate-pulses
   ;;Set State to refractory
   set nneuronstate neuron_state_refractory
   ;;initialize counter of refractory period in number of iterations
   set nrefractorycounter nintervrefractoryperiod
end 


;;;
;;; Kernel for inhibitory post-synaptic potential
;;;

to-report ipsp-kernel ;;Called by Neurons

  report 1
end 


;;;
;;; Kernel for excitatory post-synaptic potential
;;;

to-report epsp-kernel ;;Called by Neurons

  report 1
end 


;;;
;;; Kernel for membrane decay towards resting potential (If current membrane pot. > Resting pot.)                  ;;;;;;;Possibility for leak? w next
;;;

to-report negative-decay-kernel ;;Called by Neurons

  report (1 + leakiness) * (exp ( - ( ticks - nlastspikeinput ) / 5 ) + ndecayrate)
end 


;;;
;;; Kernel for membrane decay towards resting potential (If current membrane pot. < Resting pot.)
;;;

to-report positive-decay-kernel ;;Called by Neurons

  report (1 + leakiness)*(exp (3 - ( ticks - nlast-firing-time ) ^ 0.8) + 0.3)
end 


;;;
;;; Bring membrane potential towards its resting state
;;;

to decay ;;Called by Neurons
    ;;Move membrane potential towards resting potential:
    ifelse (nmembranepotential > nrestingpotential )
    [
        let expdecay negative-decay-kernel ;
        ifelse ((nmembranepotential - expdecay) < nrestingpotential)
        [
          set nmembranepotential nrestingpotential
        ]
        [
          set nmembranepotential nmembranepotential - expdecay
        ]
    ]
    [
      let expdecay positive-decay-kernel ;
      ifelse ((nmembranepotential + expdecay) > nrestingpotential)
      [
        set nmembranepotential nrestingpotential
      ]
      [
        set nmembranepotential nmembranepotential + expdecay
      ]
    ]
end 


;;;
;;; Process neuron dynamics according to its machine state
;;;

to do-neuron-dynamics ;;Called by Neurons

  ifelse ( nneuronstate = neuron_state_open )
  [
    if (nmembranepotential != nrestingpotential )
    [
      ;;Check if membrane potential reached the firing threshold
      ifelse( nmembranepotential >= nfirethreshold )
      [
        prepare-pulse-firing
        set color red
      ]
      [
         ;;Move membrane potential towards resting potential:
         decay
      ]
    ]
  ]
  [
    ;;Not idle and not firing, then refractory:
    set color pink ;;Restore normal colour
    ;;Decrease timer of absolute refractory period:
    set nrefractorycounter nrefractorycounter - system_iter_unit
    ;;Set membrane potential with refractory potential:
    set nmembranepotential nrelrefractorypotential
    if ( nrefractorycounter <= 0) ;;End of absolute refractory period?
    [
      ;;Set neuron in open state:
      set nneuronstate neuron_state_open
    ]
  ]
  ;;Continue with Axonal dynamics independently of the neuron state:
  do-synaptic-dynamics
end 


;;;
;;; Delete history of incoming spikes
;;;

to empty-pulse-history-buffer ;;Called by neurons

  set nincomingspikes[]
end 


;;;
;;; Apply the Spike Timing Dependent Plasticity rule
;;;

to apply-stdp-learning-rule ;;Call by neurons

  ;Apply rule: Ap.exp(dt/Tp); if dt < 0; dt = prespt - postspt
  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  let currspikeinfo[]
  let itemcount 0
  let deltaweight 0

  while [itemcount < ( length nincomingspikes ) ]
  [
     set currspikeinfo ( item itemcount nincomingspikes ) ;;Get spike info:   prneuronid[0], snefficacy[1], excinh[2], arrivaltime[3], processedBynegHebb[4]

     ifelse ( item 2 currspikeinfo ) = excitatory_synapse ;;Is the spike coming from an excitatory synapsis?
     [
       let deltaspike ( item 3 currspikeinfo ) - nlast-firing-time
       if ( deltaspike >= nneg_time_window and deltaspike <= npos_time_window) ;;Is spike within learning window?
       [
           ;;Calculate learning factor:
          ifelse ( deltaspike <= 0 ) ;;Increase weight
          [
            set deltaweight  npos_hebb_weight * exp(deltaspike / npos_syn_change_interval )
            ask normal-synapse  ( item 0 currspikeinfo ) nneuronid  [update-synapse-efficacy deltaweight [nmax_synaptic_weight] of myself [nmin_synaptic_weight] of myself]
          ]
          [
            if (( item 4 currspikeinfo ) = false) ;;if spike has not been processed then compute Hebb rule:
            [
              set deltaweight (- nneg_hebb_weight * exp(- deltaspike / nneg_syn_change_interval )) ;;Turn positive delta into a negative weight
              set currspikeinfo replace-item 4 currspikeinfo true ;Indicate that this pulse has already been processed as a EPSP after Postsyn neuron has fired (negative hebbian)
              set nincomingspikes replace-item itemcount nincomingspikes currspikeinfo
              ask normal-synapse  ( item 0 currspikeinfo ) nneuronid [update-synapse-efficacy deltaweight [nmax_synaptic_weight] of myself [nmin_synaptic_weight] of myself]
            ]
          ]
       ]
     ]
     [
       ;;Inhibitory Synapses: Plasticity in inhibitory synapses not implemented yet

     ]
     set itemcount itemcount + 1
  ]
end 


;;;
;;; Don't store more pulses than the specified by PulseHistoryBuffSize
;;;

to check-pulse-history-buffer ;;Call by neurons

  if( length nincomingspikes > PulseHistoryBuffSize )
  [
    ;; Remove oldest pulse in the list
    set nincomingspikes remove-item 0 nincomingspikes
  ]
end 


;;;
;;; Propagate pulse to all post-synaptic neurons
;;;

to propagate-pulses ;;Call by neurons

  ;; Insert a new pulse in all synapses having the current neuron as presynaptic
  ask my-out-normal-synapses
  [
    add-pulse-job
  ]
end 


;;;
;;; Process synaptic dynamics of synapses with pre-synaptic neuron: presynneuronid
;;;

to do-synaptic-dynamics ;;Call by neurons

  ;; Process all outgoing synapses with pulses in their job-list
  ask my-out-normal-synapses with [ length joblist > 0 ]
  [
    process-pulses-queue
  ]
end 


;;;
;;; Enqueue pulse in synapse
;;;

to add-pulse-job ;;Call by link (synapse)

    ;;Add a new Pulse with its delay time at the end of the outgoing-synapse joblist
    set joblist lput synapsedelay joblist
end 


;;;
;;; Change synaptic weight
;;;

to update-synapse-efficacy [ #deltaweight #pmax_synaptic_weight #pmin_synaptic_weight] ;;Call by synapse

  ifelse ( synapseefficacy + #deltaweight ) > #pmax_synaptic_weight
  [
    set synapseefficacy #pmax_synaptic_weight
  ]
  [
    ifelse ( synapseefficacy + #deltaweight ) < #pmin_synaptic_weight
    [
      set synapseefficacy #pmin_synaptic_weight
    ]
    [
      set synapseefficacy synapseefficacy + #deltaweight
    ]
  ]
  set thickness synapseefficacy * 0.15
end 


;;;
;;; For each traveling pulse in synapse check if pulse has already arrived at the post-synaptic neuron
;;;

to process-pulses-queue ;;Call by synapse

  set joblist map [ ? - 1  ] joblist ;;Decrease all delay counters by 1 time-unit
  foreach filter [? <= 0] joblist
  [
       ;;Transmit Pulse to Postsyn Neuron:
       ask other-end [receive-pulse [presynneuronid] of myself [synapseefficacy] of myself [exc_or_inh] of myself [learningon?] of myself]
  ]
  ;;Keep only "traveling" pulses in the list :
  set joblist filter [? > 0] joblist
end 


;;;
;;; Create one input neuron and attach it to neuron with label #ppostsynneuronlabel (input neurons have one connection only)
;;;

to setup-input-neuron [#pposx #pposy #label #ppostsynneuronlabel #psynapseefficacy #pcoding #pnumofspikes]

  let postsynneuronid get-neuronid-from-label #ppostsynneuronlabel
  set-default-shape inputneurons "square"
  create-inputneurons 1
  [
     set layernum 0
     set neuronid who
     set neuronstate neuron_state_open
     set pulsecounter 0
     set interspikecounter 0
     set numberofspikes #pnumofspikes
     set postsynneuron postsynneuronid
     set encodedvalue input_value_empty
     set isynapseefficacy #psynapseefficacy
     setxy #pposx #pposy
     set color green
     set label #label
     set neuronlabel #label
     setup-input-synapse
  ]
end 


;;;
;;; Process pulses in input neuron
;;;

to do-input-neuron-dynamics ;;Called by inputneurons

  if ( pulsecounter > 0 ) ;;process only if input-neuron has something to do
  [
    set interspikecounter interspikecounter + 1
    if (interspikecounter > encodedvalue)
    [
    ;;Transmit pulse to Post-synaptic Neuron;
      ask out-input-synapse-neighbors [receive-input-neuron-pulse [isynapseefficacy] of myself [excitatory_synapse] of myself]
      set interspikecounter 0
      set pulsecounter pulsecounter - 1
    ]
  ]
end 


;;;
;;; Encode input value (integer number) into pulses
;;;

to-report set-input-value [#pencodedvalue] ;;Called by inputneurons

  ;;Check if input neuron is ready to receive input
  let inputready false
  if ( pulsecounter = 0 )
  [
    set encodedvalue #pencodedvalue
    set pulsecounter numberofspikes ;;Initialize counter with the number of pulses to transmit with the encoded value
    set interspikecounter 0
    set inputready true
  ]
  report inputready
end 


;;;
;;; Ask input neuron with id = #pneuronid to accept and encode a new input value
;;;

to feed-input-neuron [#pneuronid #pencodedvalue];;Called by observer

   ask inputneuron #pneuronid
   [
     let inputready set-input-value #pencodedvalue
   ]
end 


;;;
;;; Ask input neuron with label = #pneuronlabel to accept and encode a new input value
;;;

to feed-input-neuron_by_label [#pneuronlabel #pencodedvalue];;Called by observer

   ask one-of inputneurons with [ neuronlabel = #pneuronlabel ]
   [
     let inputready set-input-value #pencodedvalue
   ]
end 


;;;
;;; Create link to represent synapse from input neuron to post-synaptic neuron: postsynneuron
;;;

to setup-input-synapse ;;Call from inputneurons

   let psnneuron postsynneuron
   let postsynneu one-of (normalneurons with [nneuronid = psnneuron])
   create-input-synapse-to postsynneu
end 


;;;
;;; Create and initialize neuron
;;;

to setup-normal-neuron [#playernum #pposx #pposy #label #pneurontypeid]

  set-default-shape normalneurons "circle"
  let returned_id nobody
  create-normalneurons 1
  [
     set nlayernum #playernum
     set nneuronid who
     set nneuronstate neuron_state_open
     set nrefractorycounter 0
     set nincomingspikes[]
     set nnumofsynapses 0
     set nlastspikeinput 0
     setxy #pposx #pposy
     set color pink
     set label #label
     set nneuronlabel #label
     set returned_id nneuronid
  ]
  set-neuron-to-neurontype #pneurontypeid returned_id
end 


;;;
;;; Process neural dynamics
;;;

to do-network-dynamics

 ask inputneurons [ do-input-neuron-dynamics ] ;Process input neurons in random order
 ask normalneurons [do-neuron-dynamics] ;Process normal neurons in random order
end 


;;;
;;; Show information about neuron with id: #srcneuron
;;;

to show-neuron-info [#srcneuron] ;;Called by observer

   ;set plot-list lput ( [nmembranepotential] of one-of normalneurons with [nneuronid = neuronid_monitor1] ) plot-list
   ask normalneurons with [nneuronid = #srcneuron]
   [
      print ""
      write "Neuron Id:" write nneuronid print ""
      write "Layer " write nlayernum print ""
      write "Membrane potential: " write nmembranepotential print ""
      write "Spike threshold: " write nfirethreshold print ""
      write "Resting potential: " write nrestingpotential print ""
      write "Refractory potential: " write nrelrefractorypotential print ""
      write "Last input-pulse received at:" write nlastspikeinput print ""
      write "Last spike fired at: " write nlast-firing-time print ""
      write "Current state: " write (ifelse-value ( nneuronstate = neuron_state_open ) ["idle"] ["refractory"] ) print ""
      print ""
   ]
end 


;;;
;;; Show information about synapse with pre-synaptic neuron: #srcneuron and post-synaptic neuron: #dstneuron
;;;

to show-synapse-info-from-to [#srcneuron #dstneuron] ;;Called by observer

   ask normal-synapses with [presynneuronid = #srcneuron and possynneuronid = #dstneuron]
   [
      print "Synapse from:"
      write "Neuron " write #srcneuron write " to neuron " write #dstneuron print ""
      write "Weight: " write synapseefficacy print ""
      write "Delay: " write synapsedelay write "iteration(s)" print ""
      write "Excitatory or Inhibitory: "
      write (ifelse-value ( exc_or_inh = excitatory_synapse ) ["Excitatory"] ["Inhibitory"] )
      print ""
   ]
end 

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;End of SNN code;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Artificial insect related code:;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
breed [testcreatures testcreature]
breed [visualsensors visualsensor]

testcreatures-own [

  creature_label
  creature_id
  reward_neuron
  pain_neuron
  move_neuron
  rotate_neuron
  creature_sightline

]

;;;
;;; Create insect agent
;;;

to-report create-creature [#xpos #ypos #creature_label #reward_neuron_label #pain_neuron_label #move_neuron_label #rotate_neuron_label]

  let reward_neuron_id get-input-neuronid-from-label #reward_neuron_label
  let pain_neuron_id get-input-neuronid-from-label #pain_neuron_label
  let move_neuron_id get-neuronid-from-label #move_neuron_label
  let rotate_neuron_id get-neuronid-from-label #rotate_neuron_label
  let returned_id nobody
  create-testcreatures 1 [
    set shape "bug"
    setxy #xpos #ypos
    set size 2
    set color yellow
    rt random-float 360
    set creature_label #creature_label
    set reward_neuron reward_neuron_id
    set pain_neuron pain_neuron_id
    set move_neuron move_neuron_id
    set rotate_neuron rotate_neuron_id
    set creature_id who
    set returned_id creature_id

  ]
  report  returned_id
end 


visualsensors-own [

  sensor_id
  perceived_stimuli
  distance_to_stimuli
  relative_rotation ;;Position relative to front
  attached_to_colour
  attached_to_neuron
  attached_to_creature

]


;;;
;;; Create photoreceptor and attach it to insect
;;;

to create-visual-sensor [ #psensor_id #pposition #colour_sensitive #attached_neuron_label #attached_creature] ;;Called by observer

  let attached_neuron_id get-input-neuronid-from-label #attached_neuron_label
  create-visualsensors 1 [
     set sensor_id #psensor_id
     set relative_rotation #pposition ;;Degrees relative to current heading - Left + Right 0 Center
     set attached_to_colour #colour_sensitive
     set attached_to_neuron attached_neuron_id
     set attached_to_creature #attached_creature
     ht
  ]
end 



;;;
;;; Ask photoreceptor if there is a patch ahead (within insect_view_distance) with a perceivable colour (= attached_to_colour)
;;;

to view-world-ahead ;;Called by visualsensors

  let itemcount 0
  let foundobj black
  ;;;;;;;;;;;;;;;Take same position and heading of creature:;;;;;;;;;;;;;;;
  let creature_px 0
  let creature_py 0
  let creature_heading 0
  ask  testcreature attached_to_creature [set creature_px xcor set creature_py ycor set creature_heading heading];
  set xcor creature_px
  set ycor creature_py
  set heading creature_heading
  rt relative_rotation
  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  let view_distance insect_view_distance
  let xview 0
  let yview 0
  while [itemcount <= view_distance and foundobj = black]
  [
    set itemcount itemcount + 0.2;1
    ask patch-ahead itemcount [
       set foundobj pcolor
       set xview pxcor
       set yview pycor
    ]

  ]

  update-creature-sightline-position attached_to_creature xview yview
  ifelse (foundobj = attached_to_colour) ;;Found perceivable colour?
  [
    set distance_to_stimuli itemcount
    set perceived_stimuli foundobj                                                    ;;;;;;Set percieved stimuli? Drug perception
  ]
  [
    set distance_to_stimuli 0
    set perceived_stimuli 0
  ]
end 

;;;
;;; Process Nociceptive, reward and visual sensation
;;;

to perceive-world ;;Called by testcreatures

 let nextobject 0
 let distobject 0
 let onobject 0
 ;; Get color of current position
 ask patch-here [ set onobject pcolor ]
 ifelse (onobject = white)
    [ if white_reward > 0
      [
        feed-input-neuron reward_neuron white_reward ;;induce happiness '
      ]                                       ;;ouch to drug (withdrawal symptoms?)
        ask patch-here [ set pcolor black ] ;;Eat patch
    ]
    [
      if (onobject = green)
      [ if green_reward > 0
        [
             feed-input-neuron reward_neuron green_reward ;;induce happiness
        ]
             if white_withdrawal > 0
             [
               feed-input-neuron pain_neuron white_withdrawal
             ]
             ask patch-here [ set pcolor black ]  ;;Eat patch
      ]
    ]
 ask visualsensors [propagate-visual-stimuli]
end 

;;;
;;; Move or rotate according to the active motoneuron
;;;

to do-actuators ;;Called by Creature

 let dorotation? false
 let domovement? false
 ;;Check rotate actuator
 ask normalneuron rotate_neuron [
   if (nlast-firing-time = ticks);
   [
     set dorotation? true
   ]
 ]
 ;;Check move forward actuator
 ask normalneuron move_neuron[
   if (nlast-firing-time = ticks);
   [
     set domovement? true
   ]
 ]

 if (dorotation?)
 [
   rotate-creature 4
 ]

 if (domovement?)
 [
   move-creature 0.4
 ]
end 

;;;
;;; Photoreceptor excitates the connected input neuron
;;;

to propagate-visual-stimuli ;;Called by visual sensor

  if (attached_to_colour = perceived_stimuli) ;;Only produce an action potential if the corresponding associated stimulus was sensed
  [
     feed-input-neuron attached_to_neuron distance_to_stimuli;
  ]
end 

;;;
;;; Move insect (#move_units) patches forward
;;;

to move-creature [#move_units]
  if (leave_trail_on?) [Leave-trail]
  fd #move_units
end 

;;;
;;; Rotate insect (#rotate_units) degrees
;;;

to rotate-creature [#rotate_units]
  rt #rotate_units
end 

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;End of insect related code;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;


;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
breed [sightlines sightline]
directed-link-breed [sight-trajectories sight-trajectory]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;; Show a sightline indicating the patch the insect is looking at
;;;

to-report create-sightline

  let sightline_id nobody
  create-sightlines 1
  [
     set shape "circle"
     set size 0.5
     set color orange
     set sightline_id who
     ht
  ]
  report sightline_id
end 

to update-creature-sightline-position [#creatureid #posx #posy]

  ifelse (show_sight_line?)
  [
    let attached_sightline 0
    ask testcreature #creatureid [set attached_sightline creature_sightline]
    ask sightline attached_sightline [setxy #posx #posy]
    ask sight-trajectories [show-link]
  ]
  [
    ask sight-trajectories [hide-link]
  ]
end 

to attach-sightline-to-creature [#creature_id #sightline_id]

  let sightline_agent sightline #sightline_id
  ask sightline_agent [setxy [xcor] of testcreature #creature_id [ycor] of testcreature #creature_id]
  ask testcreature #creature_id [
    set creature_sightline #sightline_id
    create-sight-trajectory-to sightline_agent [set color orange set thickness 0.4]
  ]
end 

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
breed [itrails itrail]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;
;;; Leave a yellow arrow behind the insect indicating its heading
;;;

to leave-trail ; [posx posy]

  hatch-itrails 1
  [
     set shape "arrow"
     set size 1
     set color yellow
     set ttl 2000
  ]
end 

;;;
;;; Check if it is time to remove the trail
;;;

to check-trail

  set ttl ttl - 1
  if ttl <= 0
  [
     die
  ]
end 

itrails-own
[
  ttl
]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;



globals [
  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;SNN Module globals;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  neuron_state_open;; describe state machine of the neuron
  neuron_state_firing;; describe state machine of the neuron
  neuron_state_refractory;; describe state machine of the neuron
  excitatory_synapse
  inhibitory_synapse
  system_iter_unit ;;time steps of each iteration expressed in milliseconds
  plot-list
  plot-list2
  PulseHistoryBuffSize ;;Size of pulse history buffer
  input_value_empty ;;Indicate that there is no input value
  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Insect globals;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  pspikefrequency ;;Number of spikes emitted by an input neuron in response to one stimulus
  learningmode?
  error_free_counter ;;Number of iterations since the insect collided with a noxious stimulus
  required_error_free_iterations ;;Number of error-free iterations necessary to stop training
  white_initial
  green_initial  ;; For counting plots

]


;;;
;;; Create neural circuit, insect and world
;;;

to setup

  clear-all

  RESET-TICKS

  initialize-global-vars

  ;;;;;;;;;;Draw world with white, green and red patches;;;;;;;;
  draw-world
  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  ;;;;;;;;;;;;;;Setup Neuron types;;;;;;;;;;;;;;                 ;;;change for weight redistribution?
  ;;;;;;;;;;;;;;;;Neuron type1:
  setup-neurontype 1 (-65 + 50 * leakiness) -55 0.1 -75 3 -75 ;;22,23,11,12,13,31,32
  set-neurontype-learning-params 1 0.045 75 0.045 -50 maxse 1 (8 + 3) (15 + 3) ;[ #pneurontypeid #ppos_hebb_weight #ppos_time_window #pneg_hebb_weight #pneg_time_window #pmax_synaptic_weight
                                    ;#pmin_synaptic_weight #ppos_syn_change_interval #pneg_syn_change_interval]
  ;;;;;;;;;;;;;;;;Neuron type2:
  setup-neurontype 2 (-65 + 50 * leakiness) -55 0.1 -70 1 -70 ;(typeid restpot threshold decayr refractpot refracttime)
  set-neurontype-learning-params 2 0.09 55 0.09 -25 9 1 8 15

  ;;;;;;;;;;;;;;;;Create the Neural circuit (brain) of the insect

  ;;;;;;;;;;;;;;Layer 1: Afferent neurons with receptors ;;;;;;;;;;;;;;;
  setup-normal-neuron 1 15 10  11 1 ;;[layernum pposx pposy pid pneurontypeid]                                                            ;;;visual
  setup-input-neuron 5 10  1  11 20 1 pspikefrequency ;;[pposx pposy pid ppostsynneuron psynapseefficacy pcoding pnumofspikes]

  ;;setup-normal-neuron 1 15 15  12 1 ;;setup-normal-neuron [pposx pposy pid pthreshold  prestpot pdecayr prefractpot pintrefrectp]
  ;;setup-input-neuron    5 15  2  12 20 1 pspikefrequency ;;[pposx pposy pid ppostsynneuron ipsynapseefficacy pcoding pnumofspikes]       ;;; remove red

  setup-normal-neuron 1 15 20  13 1 ;;setup-normal-neuron [pposx pposy pid pthreshold  prestpot pdecayr prefractpot pintrefrectp]
  setup-input-neuron    5 20  3  13 20 1 pspikefrequency ;;[pposx pposy pid ppostsynneuron ipsynapseefficacy pcoding pnumofspikes]

  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

  ;;;;;;;;;;;;;;; Layer 2: First hidden layer ;;;;;;;;;;;;;;;;;;;;;;                                                                      ;;; main processor
  ;Motoneuron with rotate actuator:
  setup-normal-neuron 2 25 14  21 1
  setup-input-neuron   25 5  4  21 20 1 pspikefrequency
  ;Motoneuron with move actuator:
  setup-normal-neuron 2 25 19  22 1
  setup-input-neuron   25 30  5  22 20 1 pspikefrequency

  ;;;;;;;;;;;;;; Synapses from Layer 1 to Layer 2 ;;;;;;;;;;;;;;;;;;
  ;;Synapse from afferent neuron 1001 to Motoneurons:
  setup-synapse   11  21 5 excitatory_synapse 3 true
  setup-synapse   11  22 5 excitatory_synapse 3 true

  ;;Synapse from afferent neuron 1002 to Motoneurons:                         ;;remove red
 ;; setup-synapse   12  21 5 excitatory_synapse 3 true
  ;;setup-synapse   12  22 5 excitatory_synapse 3 true

  ;;Synapse from afferent neuron 1003 to Motoneurons:
  setup-synapse   13  21 5 excitatory_synapse 3 true
  setup-synapse   13  22 5 excitatory_synapse 3 true

  ;;;;;;;;;;;;;;  Layer 3: Output layer ;;;;;;;;;;;;;;;;;;
  ;;Actuator move forward:
  setup-normal-neuron 3 35 19  31 1
  ;;Actuator rotate:
  setup-normal-neuron 3 35 14  32 1

  ;;;;;;;;;;;;; Synapses from Layer 2 to Layer 3 ;;;;;;;;;;;;;;;;;;;;

  ;;Mutual inhibitory synapses between Motoneurons:
  setup-synapse  21  22 22 inhibitory_synapse 3 true
  setup-synapse  22  21 8 inhibitory_synapse 3 true
  ;;Positive Synapsis from Nociceptive Motoneuron to rotate actuator (no plasticity):
  setup-synapse  21  32 11 excitatory_synapse 3 false
  ;;Positive Synapsis from Reward Motoneuron to move forward actuator (no plasticity):
  setup-synapse  22  31 11 excitatory_synapse 3 false

  ;;;;;;;;;;;;;;;;;;;; Oscillator (Pacemaker) ;;;;;;;;;;;;;;;;;;;;;;;
  setup-normal-neuron 2 16 25  23 2
  setup-normal-neuron 2 22 25  24 2
  setup-synapse  23  24 15 excitatory_synapse 2 false ;(no plasticity needed)
  setup-synapse  24  23 15 excitatory_synapse 3 false ;(no plasticity needed)
  setup-input-neuron 11 25  6  23 20 1 pspikefrequency ;;Voltage clamp to start pacemaker
  ;;Synapse from Pacemaker to Reward Motoneuron:
  setup-synapse  23  22 3.5 excitatory_synapse 3 false ;4.62

  ;; Start insect hearth
  feed-input-neuron_by_label  6 1

  ask patches with [ pxcor = 102 and pycor = 46 ] [set pcolor black]
  let creatureid create-creature 102 30 1  5  4  31  32;;[#xpos #ypos #creature_id #reward_neuron #pain_neuron #move_neuron #rotate_neuron]

  ;;;;;;;;;;Create Visual sensors;;;;;;;;;
  create-visual-sensor  1 0  white  1 creatureid;[ psensor_id pposition colour_sensitive attached_neuron attached_creature]
  ;create-visual-sensor  2 0 red  2 creatureid;[ psensor_id pposition colour_sensitive attached_neuron attached_creature]                   ;;; remove red
  create-visual-sensor  3 0 green  3 creatureid;[ psensor_id pposition colour_sensitive attached_neuron attached_creature]

  ;;;;;;;;;;Create Sightline ;;;;;;;;;;;;;
  let sightlineid create-sightline
  attach-sightline-to-creature creatureid sightlineid

  set white_initial count patches with [pcolor = white]  ;; initial count of patches
  set green_initial count patches with [pcolor = green]
  set learningmode? true ;; initiate learning
end 

;;;
;;; Set gloval variables with their initial values
;;;

to initialize-global-vars

  set system_iter_unit 1 ;; each simulation iteration represents 1 time unit

  set neuron_state_open 1 ;;State machine idle

  set neuron_state_firing 2 ;;State machine firing

  set neuron_state_refractory 3 ;;State machine refractory

  set excitatory_synapse 1

  set inhibitory_synapse 2

  set plot-list[] ;;List for spike history

  set plot-list2[] ;;List for spike history

  set PulseHistoryBuffSize 30

  set input_value_empty -1

  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Insect globals;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  set pspikefrequency 1
  set error_free_counter 0
  set required_error_free_iterations 35000
end 


;;;
;;; Generate insect world with 3 types of patches
;;;

to draw-world

   ask patches with [ pxcor >= 80 and pxcor <= 120 and pycor >= 1 and pycor <= 60 ]
   [
     set pcolor black
   ]
   ask n-of white_number patches with [ pxcor >= 80 and pxcor <= 120 and pycor >= 1 and pycor <= 60 ]
     [
       set pcolor white
     ]
   ask n-of green_number patches with [  pxcor >= 80 and pxcor <= 120 and pycor >= 1 and pycor <= 60 and pcolor != white ]
     [
       set pcolor green
     ]
end 

;;;
;;; Don't allow the insect to go beyond the world boundaries
;;;

to check-boundaries

      ask testcreatures [
        if (xcor < 80 or xcor > 120) or (ycor < 1 or ycor > 60)
        [
          setxy 102 30
          rt random-float 360
        ]
      ]
end 

to check-learning                                               ;;turn learning off eventually (keep?)
   if learningmode?
   [
     if error_free_counter >= required_error_free_iterations
     [
       set learningmode? true ;;false
     ]
   ]
end 


;;;
;;; Run simulation
;;;

to go

   ask itrails [ check-trail ]
   ask visualsensors [ view-world-ahead ] ;;Feed visual sensors at first
   ask testcreatures [ perceive-world]; do-actuators]
   do-network-dynamics
   ask testcreatures [do-actuators]

 check-boundaries
 tick
end