Chaines de markov

with open("./train.ester1.cut.bio") as f:
    content = f.readlines()
    
content = [ x.strip().split(" ") for x in content]

mots_differents = {}
etiquettes_differentes = {}

for x in content:
    
    if x[0] == '':
        continue
        
    if(x[0] in mots_differents):
        mots_differents[x[0]] = mots_differents[x[0]] + 1
    else:
        mots_differents[x[0]] = 1
        
    if(x[1] in etiquettes_differentes):
        etiquettes_differentes[x[1]] = etiquettes_differentes[x[1]] + 1
    else:
        etiquettes_differentes[x[1]] = 1
                

print("il existe", len(mots_differents), "mots differents")
triMot = sorted(mots_differents, key=mots_differents.get, reverse=True)
# sorted(mots_differents.values())
print("Les 10 mots les plus frequents sont", triMot[:10])
print("Les 10 mots les moins frequents sont", triMot[-10:])

il existe 33085 mots differents
Les 10 mots les plus frequents sont ['de', 'la', 'le', "l'", 'à', 'et', 'les', 'des', 'est', "d'"]
Les 10 mots les moins frequents sont ['couronner', 'irrattrapable', 'orphelines', 's(ix)', 'Au(rillac)', 'Aurillac', 'verglaçante', 'envahissants', 'radoucir', 'généralisera']

etiquettes = etiquettes_differentes.copy()

size_eti_bet = 1
if "O" in etiquettes_differentes:
    del etiquettes_differentes["O"]
    
etiquettes_join_beg = {}

for eti, count in etiquettes_differentes.items():
    if eti[0] == "B":
        etiquettes_join_beg[eti[2:]] = count
        size_eti_bet += count
        
size_eti_inn = 1
etiquettes_join_inn = {}
for eti, count in etiquettes_differentes.items():
    if eti[0] == "I":
        etiquettes_join_inn[eti[2:]] = count
        size_eti_inn += count
    
    
print("Il y a", len(etiquettes_differentes), "etiquettes differents\n")
print("Les etiquette begin: ", etiquettes_join_beg)
print("Les etiquette inner: ", etiquettes_join_inn)

Il y a 14 etiquettes differents

Les etiquette begin:  {'org': 12449, 'time': 8717, 'func': 3314, 'pers': 13647, 'loc': 15103, 'amount': 3007, 'prod': 782}
Les etiquette inner:  {'org': 9299, 'time': 9688, 'func': 5995, 'pers': 12174, 'loc': 5562, 'amount': 5255, 'prod': 1574}

import math

def probaInitiale(train):
    etiquettes_initiales = {}
    nombre_etiquette = 0
    deb = False
    for item in train:
        if(item[0] == ''):
            deb = True
        if deb and item[0] != '':
            nombre_etiquette += 1
            if(item[1] in etiquettes_initiales):
                etiquettes_initiales[item[1]] = etiquettes_initiales[item[1]] + 1
            else:
                etiquettes_initiales[item[1]] = 1
            deb = False
            
    for key, value in etiquettes_initiales.items():
        etiquettes_initiales[key] = value/nombre_etiquette
    return etiquettes_initiales
            
probaInitiale(content)

{'B-amount': 0.0024563499966806083,
 'B-func': 0.002469627564230233,
 'B-loc': 0.008019650799973445,
 'B-org': 0.014538936466839274,
 'B-pers': 0.03505277833100976,
 'B-prod': 0.0010622054039699928,
 'B-time': 0.009042023501294563,
 'O': 0.9273584279360021}

class Etat():
    def __init__(self, name):
        self.out = {}
        self.name = name
        self.transi = {}
        
    def addOneToOut(self, word):
        if word not in self.out:
            self.out[word] = 1
        else:
            self.out[word] += 1
            
    def addOneToTransi(self, otherEtat):
        if otherEtat.name not in self.transi:
            self.transi[otherEtat.name] = [1, otherEtat]
        else:
            self.transi[otherEtat.name][0] += 1
    
    def __str__(self):
        
        from collections import OrderedDict
        triMot = OrderedDict(sorted(self.out.items(), key=lambda t: -t[1]))
        motPrint = list(triMot.items())[:3]
        return 'name: ' + self.name + ',\n  out: ' + str(motPrint) + ',\n  transi: ' + str(self.transi.values())
    
    def calculateProbaTransi(self):
        total = 0
        for x in self.transi.values():
            total += x[0]
            
        for key in self.transi.keys():
            self.transi[key][0] = self.transi[key][0]/total
            
    def calculateProbaEmission(self):
        total = 0
        for x in self.out.values():
            total += x
            
        for key in self.out.keys():
            self.out[key] = self.out[key]/total
            
    def getProbaFor(self, word):
        if word not in self.out:
            return 0
        return self.out[word]
    
    def getEtatForNext(self, word):
        allProbaForEtat = ([ ( eta[1], eta[0] * eta[1].getProbaFor(word) ) for eta in self.transi.values() ])
        return max(allProbaForEtat, key=lambda x: x[1])
                  
            
    def __repr__(self):
        return self.name
        
a = Etat("B-org")
b = Etat("I-org")
c = Etat("B-test")

b.addOneToOut('out')
b.addOneToOut('out')

b.addOneToOut('out2')
b.addOneToOut('out2')
b.addOneToOut('out2')

a.addOneToOut('test')
a.addOneToOut('test')
a.addOneToOut('test')

a.addOneToOut('test2')
a.addOneToOut('test2')

a.addOneToTransi(a)

a.addOneToTransi(c)

a.addOneToTransi(b)
a.addOneToTransi(b)
a.addOneToTransi(b)

a.calculateProbaTransi()
a.calculateProbaEmission()

b.calculateProbaEmission()
b.calculateProbaEmission()

print(a.getProbaFor('test'))
print(a.getEtatForNext('out'))

print(a)

0.6
(I-org, 0.24)
name: B-org,
  out: [('test', 0.6), ('test2', 0.4)],
  transi: dict_values([[0.2, B-org], [0.2, B-test], [0.6, I-org]])

# init Etats
all_etats = {}
for eti, _ in etiquettes.items():
    all_etats[eti] = (Etat(eti))
    
for key, eti in all_etats.items():
    if key[0] == 'B':
        eti.addOneToTransi(all_etats[ 'I-' + key[2:] ])
        for key2, eti2 in all_etats.items():
            if key2[0] != "I" and key != key2:
                eti2.addOneToTransi(eti)
                
    if key[0] == 'O':
        eti.addOneToTransi(all_etats[key])
                
    if key[0] == 'I':
        eti.addOneToTransi(all_etats[key])
        for key2, eti2 in all_etats.items():
            if key2[0] != "I" and key != key2:
                eti.addOneToTransi(all_etats[ key2 ])
                
    
# Train 
for i in range(len(content)-1):
    input_etat = content[i]
    input_etat_next = content[i+1]
    
    if input_etat[0] != '':
        all_etats[input_etat[1]].addOneToOut(input_etat[0])
    
    if input_etat[0] != '' and input_etat_next[0] != '':
        all_etats[input_etat[1]].addOneToTransi(all_etats[input_etat_next[1]])
        
# print and caculate Proba
for key, etat in all_etats.items():
    etat.calculateProbaEmission()
    etat.calculateProbaTransi()
    print(etat, "\n")

name: O,
  out: [('de', 0.05137359980274979), ('la', 0.03188833022518945), ('le', 0.027748527775091167)],
  transi: dict_values([[0.9306113315982241, O], [0.015826492998716333, B-org], [0.011067209967849446, B-time], [0.00445161518259866, B-func], [0.012347932589827235, B-pers], [0.020794813515009482, B-loc], [0.0039128284243873145, B-amount], [0.000987775723387467, B-prod]]) 

name: B-org,
  out: [('France', 0.09093099847377299), ('États', 0.02980159048919592), ('Front', 0.02474094304763435)],
  transi: dict_values([[0.548695652173913, I-org], [0.0037681159420289855, B-time], [9.66183574879227e-05, B-func], [0.012753623188405797, B-pers], [0.0001932367149758454, B-loc], [0.0001932367149758454, B-amount], [0.0001932367149758454, B-prod], [0.418743961352657, O], [0.01536231884057971, B-org]]) 

name: I-org,
  out: [('de', 0.08140660286052263), ('Inter', 0.05860845252177654), ('Unis', 0.04000430153779976)],
  transi: dict_values([[0.5123779884000565, I-org], [0.42792474183052764, O], [0.013014570660630925, B-org], [0.016975526948649032, B-time], [0.00014146272457207527, B-func], [0.026312066770405998, B-pers], [0.0014146272457207527, B-loc], [0.0014146272457207527, B-amount], [0.0004243881737162258, B-prod]]) 

name: B-time,
  out: [("aujourd'hui", 0.13685901112768153), ('hier', 0.13009062750946426), ('deux', 0.05712974647241023)],
  transi: dict_values([[0.0011522212264754833, B-org], [0.5832799897580335, I-time], [0.00038407374215849446, B-func], [0.008065548585328383, B-pers], [0.0005120983228779926, B-loc], [0.0002560491614389963, B-amount], [0.0005120983228779926, B-prod], [0.3981564460376392, O], [0.007681474843169889, B-time]]) 

name: I-time,
  out: [('heures', 0.10322047894302229), ('neuf', 0.06430635838150289), ('mille', 0.06306771263418662)],
  transi: dict_values([[0.6570258510366009, I-time], [0.3111082672126952, O], [0.004863066291272076, B-org], [0.011517788584591758, B-time], [0.0001279754287176862, B-func], [0.014077297158945483, B-pers], [0.000639877143588431, B-loc], [0.0005119017148707448, B-amount], [0.0001279754287176862, B-prod]]) 

name: B-func,
  out: [('président', 0.2700663850331925), ('ministre', 0.18135184067592033), ('Premier', 0.14725407362703682)],
  transi: dict_values([[0.001244167962674961, B-org], [0.0006220839813374805, B-time], [0.7228615863141524, I-func], [0.1359253499222395, B-pers], [0.001244167962674961, B-loc], [0.00031104199066874026, B-amount], [0.00031104199066874026, B-prod], [0.0009331259720062209, B-func], [0.136547433903577, O]]) 

name: I-func,
  out: [('de', 0.09991659716430358), ('ministre', 0.08473728106755629), ('la', 0.06422018348623854)],
  transi: dict_values([[0.6771755162241888, I-func], [0.16371681415929204, O], [0.0003687315634218289, B-org], [0.0011061946902654867, B-time], [0.0014749262536873156, B-func], [0.1544985250737463, B-pers], [0.0012905604719764012, B-loc], [0.00018436578171091445, B-amount], [0.00018436578171091445, B-prod]]) 

name: B-pers,
  out: [('Jean', 0.039056202828460465), ('Saddam', 0.022056129552282554), ('Mohamed', 0.019564739503187514)],
  transi: dict_values([[0.00023749208359721342, B-org], [0.0005541481950601647, B-time], [0.00015832805573147562, B-func], [0.8180018999366688, I-pers], [0.00015832805573147562, B-loc], [0.0018999366687777073, B-amount], [0.00023749208359721342, B-prod], [0.01266624445851805, B-pers], [0.16608613046231793, O]]) 

name: I-pers,
  out: [('Hussein', 0.02053556760308855), ('Bush', 0.020206998521439132), ('Pen', 0.018317726301954985)],
  transi: dict_values([[0.23014485514485514, I-pers], [0.6896853146853147, O], [0.0241008991008991, B-org], [0.004995004995004995, B-time], [0.011488511488511488, B-func], [0.027097902097902096, B-pers], [0.00924075924075924, B-loc], [0.0014985014985014985, B-amount], [0.0017482517482517483, B-prod]]) 

name: B-loc,
  out: [('Irak', 0.053966362071248844), ('Paris', 0.0372136140908489), ('France', 0.03688253211495166)],
  transi: dict_values([[0.009377159214292765, B-org], [0.011153884118053499, B-time], [9.870693909781858e-05, B-func], [0.03760734379626888, B-pers], [0.3055966834468463, I-loc], [0.002961208172934557, B-amount], [0.00019741387819563716, B-prod], [0.6100088836245188, O], [0.02299871680979173, B-loc]]) 

name: I-loc,
  out: [('de', 0.12333692916217188), ('Orient', 0.06005034160373966), ("d'", 0.039554117224020136)],
  transi: dict_values([[0.5855278766310795, I-loc], [0.37722419928825623, O], [0.004270462633451958, B-org], [0.0056939501779359435, B-time], [0.00047449584816132857, B-func], [0.014472123368920523, B-pers], [0.009252669039145907, B-loc], [0.0028469750889679717, B-amount], [0.00023724792408066428, B-prod]]) 

name: B-amount,
  out: [('vingt', 0.12736947123378783), ('trois', 0.09743930828067841), ('deux', 0.0834718989025607)],
  transi: dict_values([[0.00033829499323410016, B-org], [0.00033829499323410016, B-time], [0.00033829499323410016, B-func], [0.00033829499323410016, B-pers], [0.00033829499323410016, B-loc], [0.9248985115020297, I-amount], [0.00033829499323410016, B-prod], [0.009133964817320704, B-amount], [0.06393775372124492, O]]) 

name: I-amount,
  out: [('ans', 0.11189343482397716), ('cinq', 0.04205518553758325), ('degrés', 0.04072312083729781)],
  transi: dict_values([[0.5679873930661864, I-amount], [0.4090499774876182, O], [0.00022512381809995497, B-org], [0.0036019810895992796, B-time], [0.00022512381809995497, B-func], [0.007203962179198559, B-pers], [0.00022512381809995497, B-loc], [0.01125619090499775, B-amount], [0.00022512381809995497, B-prod]]) 

name: B-prod,
  out: [('Déclaration', 0.0741687979539642), ('PACS', 0.06265984654731457), ('Achille', 0.03964194373401535)],
  transi: dict_values([[0.0013812154696132596, B-org], [0.0027624309392265192, B-time], [0.0013812154696132596, B-func], [0.006906077348066298, B-pers], [0.004143646408839779, B-loc], [0.0027624309392265192, B-amount], [0.761049723756906, I-prod], [0.2154696132596685, O], [0.004143646408839779, B-prod]]) 

name: I-prod,
  out: [('de', 0.0940279542566709), ('des', 0.05717916137229987), ("l'", 0.05717916137229987)],
  transi: dict_values([[0.7492690058479532, I-prod], [0.22514619883040934, O], [0.0007309941520467836, B-org], [0.008771929824561403, B-time], [0.0007309941520467836, B-func], [0.007309941520467836, B-pers], [0.0014619883040935672, B-loc], [0.0007309941520467836, B-amount], [0.005847953216374269, B-prod]]) 

observations = ("les", "États", "Unis", "organisent", "d'", "aujourd'hui",
                 "au", "seize", "mai", "une", "rencontre", 
                 "avec", "le",
                "ministre", 
                "tunisien", 
                "des", 
                "affaires", 
                "étrangères", 
                 "Bertrand", "Gallicher", "a",
                 "cinq", "minutes", "de", "Casablanca", "en", "RER")


probaInit = probaInitiale(content)
for key in all_etats.keys():
    if key not in probaInit:
        probaInit[key] = 0
        
import math
def log(x):
    if x == 0:
        return -10000000
    return math.log(x)

vectorProba = []
for etat in all_etats.values():
    vectorProba.append((etat , log(etat.getProbaFor(observations[0])) + log(probaInit[etat.name])))
        
maxLast = max(vectorProba, key = lambda x: x[1])

print(maxLast, observations[0])
    
for word in observations[1:]:
    tmp = maxLast[1]
    maxLast = maxLast[0].getEtatForNext(word)
    maxLast = (maxLast[0], tmp + log(maxLast[1]) + log(maxLast[0].getProbaFor(word)))
    print(maxLast, word)

(O, -3.93450198991896) les
(B-org, -15.106958990559034) États
(I-org, -22.14470693378522) Unis
(O, -46.11447657651617) organisent
(O, -54.65122768019723) d'
(B-time, -63.13260427747703) aujourd'hui
(O, -73.69242462104295) au
(B-time, -87.19283210551181) seize
(I-time, -95.43345428944174) mai
(O, -105.77949732396587) une
(O, -121.4367272810701) rencontre
(O, -132.4168298501158) avec
(O, -139.65788839979706) le
(B-func, -148.48700921297683) ministre
(I-func, -159.8528012059041) tunisien
(I-func, -166.5970662889034) des
(I-func, -174.31034800279994) affaires
(I-func, -182.4249711076208) étrangères
(B-pers, -197.24604695378832) Bertrand
(I-pers, -213.4884644693774) Gallicher
(O, -222.56882740830713) a
(O, -236.58013978124748) cinq
(O, -256.4977974724582) minutes
(O, -262.5069727574153) de
(B-loc, -275.40121378303667) Casablanca
(O, -284.26540870781065) en
(B-prod, -303.12287871438633) RER