(defun contexts-list-to-prob-dist (contexts-list)
"A closure that converts a list of context objects to a probability distribution over words, given said entity pair."
(let ((local-hash (make-hash-table :test 'equalp))
(new-bm-vector (make-array (length *reference-vector*)))
(context-count (length contexts-list))
(mu 0))
(flet
((magical-word-counter (thiscontext)
(print thiscontext)
(let ((string-this-context (middle thiscontext)))
(unless (equalp string-this-context 0)
(map nil #'(lambda (word) (incf (gethash word local-hash 0.0))) (cl-ppcre:split "\\s+" string-this-context)))))
(calc-context-probs ()
(loop
for feature being the hash-keys of local-hash
using (hash-value feature-count)
do (setf (aref new-bm-vector (position feature *reference-vector*)) (dirichlet-calc feature feature-count mu context-count)))
(loop
for feature being the hash-keys of *corpus-fc-table* ;backoff if we've got no counts locally for a feature. Try commenting this out, see what happens.
using (hash-value feature-count)
unless (gethash feature local-hash) do (setf (aref new-bm-vector (position feature *reference-vector*)) (dirichlet-calc feature 0 mu context-count)))))
;(print contexts-list)
(map nil #'magical-word-counter contexts-list);This counts all occurances of a feature (i) in a context (j), storing the results in local-hash, above.
(setf mu (count-all-features local-hash))
(calc-context-probs)
(make-prob-dist *reference-vector* new-bm-vector))))
(defun normalize-context-matrix (context-matrix)
"The toplevel normalizer function. Takes a context matrix and destructively replaces each cell with a normalized probability."
(map-into-matrix context-matrix #'contexts-list-to-prob-dist)
(setf *norm-cm* context-matrix))
(defun map-into-matrix (matrix func)
(let ((flattened-matrix (make-array (array-total-size matrix) :displaced-to matrix)))
(map-into flattened-matrix (thereis func) flattened-matrix)
matrix))
(defun thereis (fn)
#'(lambda (&rest args)
(when (every (complement #'null) args)
(apply fn args))))