229 def parseContent(self):
232 read line from getFileContent (=> method = 'file') or getStreamContent (=> method ='stream') and extract data 233 If bmSearch (main) kwargs "action" is set and has value : 234 - "storeToELK", try to post data to an Elasticsearch instance 235 - "updateToELK", try to update an Elasticsearch instance data (deliveredRecip) 242 if self.method ==
"file":
244 if self.method ==
"stream":
249 self.cDb.updateTrigger(self.dbStoreMaxDays)
253 if self.method ==
"file":
254 for line
in self.getFileContent():
257 if self.method ==
"stream":
258 for line
in self.getStreamContent():
263 def decodev2(self, bodySubject):
266 return email subject in pretty view 268 bodySubject ; the email subject entry 270 data=self.pat2.findall(bodySubject)
274 (raw,extra1,encoding,method,string,extra)=g
275 extra1=extra1.replace(
'\r',
'').replace(
'\n',
'').strip()
278 if method.lower()==
'q':
279 string=quopri.decodestring(string)
280 string=string.replace(
"_",
" ").strip()
281 if method.lower()==
'b':
282 string=base64.b64decode(string)
283 line.append(string.decode(encoding,errors=
'ignore'))
284 extra=extra.replace(
'\r',
'').replace(
'\n',
'').strip()
292 def parseLine(self, line):
295 read line and extract data 296 If bmSearch (main) kwargs "action" is set and has value : 297 - "storeToELK", try to post data to an Elasticsearch instance 298 - "updateToELK", try to update an Elasticsearch instance data (deliveredRecip) 300 thisLine = line.rstrip()
303 smtpdLineEntries = self.smtpdLine.match(thisLine)
304 if smtpdLineEntries
is not None:
307 date = time.strftime(
'%Y-%m-%d',time.localtime())
309 date = self.dates[smtpdLineEntries.group(
'date')]
314 hour = smtpdLineEntries.group(
'hour')
315 timestp = int(datetime.strptime(date +
" " + hour,
'%Y-%m-%d %H:%M:%S').strftime(
"%s")) * 1000
316 queueid = smtpdLineEntries.group(
'queueid')
317 server = smtpdLineEntries.group(
'server').split(
",")[0].replace(
"[",
" : ").replace(
"]",
"")
319 self.data[queueid] = defaultdict(dict)
320 self.data[queueid][
'@timestamp'] = timestp
321 self.data[queueid][
'date'] = date
322 self.data[queueid][
'hour'] = hour
323 self.data[queueid][
'server'] = server
326 self.data[queueid][
'source'] = self.maillog
327 self.data[queueid][
'type'] =
'message' 329 self.data[queueid][
'beat'] = defaultdict(dict)
330 self.data[queueid][
'beat'][
'hostname'] = config.HOSTNAME
331 self.data[queueid][
'beat'][
'name'] =
'bmsearch' 332 self.data[queueid][
'beat'][
'version'] = config.VERSION
334 self.data[queueid][
'from'] = self.data[queueid][
'fromDomain'] = self.data[queueid][
'nrcpt'] = self.data[queueid][
'size'] =
"" 335 self.data[queueid][
'bodyDate'] = self.data[queueid][
'bodyFrom'] = self.data[queueid][
'bodyFromDomain'] = self.data[queueid][
'bodySubject'] =
"" 337 self.data[queueid][
'antivirus'] =
'CLEAN : not analyzed !' 339 self.data[queueid][
'filename'] = []
340 self.data[queueid][
'recip'] = []
341 self.data[queueid][
'deferredRecip'] = []
342 self.data[queueid][
'bouncedRecip'] = []
343 self.data[queueid][
'recipType'] = defaultdict(dict)
344 self.data[queueid][
'recipType'][
'internal'] = 0
345 self.data[queueid][
'recipType'][
'external'] = 0
347 self.data[queueid][
'fileType'] = defaultdict(dict)
348 for thisType
in self.theseFileTypes:
349 self.data[queueid][
'fileType'][thisType] = 0
354 cleanupLineEntries = self.cleanupLine.match(thisLine)
355 if cleanupLineEntries
is not None:
356 queueid = cleanupLineEntries.group(
'queueid')
357 messageid = cleanupLineEntries.group(
'messageid').replace(
"<",
"").replace(
">",
"")
358 self.data[queueid][
'msgid'] = messageid
361 if 'recip' in self.data[queueid].keys():
364 self.data[queueid][
'recip'] = []
365 self.data[queueid][
'recipType'] = defaultdict(dict)
366 self.data[queueid][
'recipType'][
'internal'] = 0
367 self.data[queueid][
'recipType'][
'external'] = 0
371 if self.cDb.verifyIfMessageidExists(messageid):
373 del self.data[queueid]
376 self.updateData[queueid] = defaultdict(dict)
377 self.updateData[queueid][
'deferredRecip'] = []
378 self.updateData[queueid][
'bouncedRecip'] = []
379 self.updateData[queueid][
'msgid'] = messageid
380 self.updateSurveyQueuid.append(queueid)
383 self.surveyQueuid.append(queueid)
384 self.queueidFromMessageid[messageid] = queueid
386 if '@timestamp' in self.data[queueid].keys():
387 timestamp = self.data[queueid][
'@timestamp']/1000
389 timestamp = int(time.time())
392 tupl=(messageid,timestamp)
393 self.cDb.addEntry(tupl)
399 klmsLineEntries = klmsLine.match(thisLine.replace(
"\"",
""))
400 if klmsLineEntries
is not None:
401 messageid = klmsLineEntries.group(
'messageid')
402 status = klmsLineEntries.group(
'status')
403 status +=
", antivirus-status:" + klmsLineEntries.group(
'avstatus')
404 status +=
", antiphishing-status:" + klmsLineEntries.group(
'apstatus')
405 status +=
", antispam-status:" + klmsLineEntries.group(
'asstatus')
407 if messageid
in self.queueidFromMessageid.keys():
408 queueid = self.queueidFromMessageid[messageid]
409 self.data[queueid][
'antivirus'] = status
411 del self.queueidFromMessageid[messageid]
418 amavisLineEntries = amavisLine.match(thisLine)
419 if amavisLineEntries
is not None:
420 messageid = amavisLineEntries.group(
'messageid')
421 status = amavisLineEntries.group(
'status')
423 if messageid
in self.queueidFromMessageid.keys():
424 queueid = self.queueidFromMessageid[messageid]
425 self.data[queueid][
'antivirus'] = status
427 del self.queueidFromMessageid[messageid]
434 filenameLineEntries = self.filenameLine.match(thisLine)
435 if filenameLineEntries
is not None:
436 queueid = filenameLineEntries.group(
'queueid')
437 filename = filenameLineEntries.group(
'filename')
440 if queueid
in self.surveyQueuid:
442 fileType = filename.lower().split(
'.')[-1].replace(
"jpg",
"jpeg")
444 if filename
not in self.data[queueid][
'filename']:
445 self.data[queueid][
'filename'].append(filename)
447 self.data[queueid][
'fileType'][fileType] += 1
449 self.data[queueid][
'fileType'][fileType] = 0
450 self.data[queueid][
'fileType'][fileType] += 1
460 bodyDateLineEntries = self.bodyDateLine.match(thisLine)
461 if bodyDateLineEntries
is not None:
462 queueid = bodyDateLineEntries.group(
'queueid')
463 bodyDate = bodyDateLineEntries.group(
'bodyDate')
465 self.data[queueid][
'bodyDate'] = bodyDate
470 bodyFromLineEntries = self.bodyFromLine.match(thisLine)
471 if bodyFromLineEntries
is not None:
472 queueid = bodyFromLineEntries.group(
'queueid')
473 bodyFrom = bodyFromLineEntries.group(
'bodyFrom')
475 self.data[queueid][
'bodyFrom'] = bodyFrom.lower().split(
'<')[-1].replace(
">",
"")
476 self.data[queueid][
'bodyFromDomain'] = bodyFrom.lower().split(
'@')[-1].replace(
">",
"")
481 bodySubjectLineEntries = self.bodySubjectLine.match(thisLine)
482 if bodySubjectLineEntries
is not None:
483 queueid = bodySubjectLineEntries.group(
'queueid')
484 bodySubject = bodySubjectLineEntries.group(
'bodySubject')
486 self.data[queueid][
'bodySubject'] = self.decodev2(bodySubject)
492 qmgrLineEntries = self.qmgrLine.match(thisLine)
493 if qmgrLineEntries
is not None:
494 queueid = qmgrLineEntries.group(
'queueid')
495 thisFrom = qmgrLineEntries.group(
'from')
496 size = qmgrLineEntries.group(
'size')
497 nrcpt = qmgrLineEntries.group(
'nrcpt')
502 thisFrom=
'notification@none' 505 if queueid
in self.surveyQueuid:
506 self.data[queueid][
'size'] = size
507 self.data[queueid][
'nrcpt'] = nrcpt
508 self.data[queueid][
'from'] = thisFrom.lower()
509 self.data[queueid][
'fromDomain'] = thisFrom.lower().split(
'@')[-1].replace(
">",
"")
516 smtpLineDSN2Entries = self.smtpLineDSN2.match(thisLine)
517 if smtpLineDSN2Entries
is not None:
518 queueid = smtpLineDSN2Entries.group(
'queueid')
519 recip = smtpLineDSN2Entries.group(
'recip')
520 orig = smtpLineDSN2Entries.group(
'orig')
522 recipDomain = recip.lower().split(
'@')[-1]
524 if queueid
in self.surveyQueuid:
525 if queueid
in self.data.keys():
526 if 'recipType' in self.data[queueid].keys():
529 self.data[queueid][
'recipType'] = {
'external': 0 ,
'internal': 0 }
532 if self.data[queueid][
'fromDomain'] == recipDomain :
533 self.data[queueid][
'recipType'][
'internal'] += 1
535 self.data[queueid][
'recipType'][
'external'] += 1
537 self.data[queueid][
'recip'].append(recip.lower())
543 smtpLineDSN4Entries = self.smtpLineDSN4.match(thisLine)
544 if smtpLineDSN4Entries
is not None:
545 queueid = smtpLineDSN4Entries.group(
'queueid')
546 deferredRecip = smtpLineDSN4Entries.group(
'recip')
547 orig = smtpLineDSN4Entries.group(
'orig')
549 recipDomain = deferredRecip.lower().split(
'@')[-1]
552 if queueid
in self.surveyQueuid:
553 if queueid
in self.data.keys():
555 if 'recipType' in self.data[queueid].keys():
558 self.data[queueid][
'recipType'] = {
'external': 0 ,
'internal': 0 }
561 if self.data[queueid][
'fromDomain'] == recipDomain :
562 self.data[queueid][
'recipType'][
'internal'] += 1
564 self.data[queueid][
'recipType'][
'external'] += 1
565 self.data[queueid][
'deferredRecip'].append(deferredRecip.lower())
569 if queueid
in self.updateSurveyQueuid:
570 self.updateData[queueid][
'deferredRecip'].append(deferredRecip.lower())
575 smtpLineDSN5Entries = self.smtpLineDSN5.match(thisLine)
576 if smtpLineDSN5Entries
is not None:
577 queueid = smtpLineDSN5Entries.group(
'queueid')
578 bouncedRecip = smtpLineDSN5Entries.group(
'recip')
579 orig = smtpLineDSN5Entries.group(
'orig')
582 recipDomain = bouncedRecip.lower().split(
'@')[-1]
585 if queueid
in self.surveyQueuid:
586 if queueid
in self.data.keys():
587 if 'recipType' in self.data[queueid].keys():
590 self.data[queueid][
'recipType'] = {
'external': 0 ,
'internal': 0 }
593 if self.data[queueid][
'fromDomain'] == recipDomain :
594 self.data[queueid][
'recipType'][
'internal'] += 1
596 self.data[queueid][
'recipType'][
'external'] += 1
598 self.data[queueid][
'bouncedRecip'].append(bouncedRecip.lower())
602 if queueid
in self.updateSurveyQueuid:
603 self.updateData[queueid][
'bouncedRecip'].append(bouncedRecip.lower())
608 smtpRemovedEntries = self.smtpRemoved.match(thisLine)
609 if smtpRemovedEntries
is not None:
610 queueid = smtpRemovedEntries.group(
'queueid')
613 if queueid
in self.surveyQueuid:
614 if self.action ==
"storeToELK":
619 del self.data[queueid]
624 if queueid
in self.updateSurveyQueuid:
625 if self.action ==
"storeToELK":
627 if len(self.updateData[queueid][
'bouncedRecip'])
or len(self.updateData[queueid][
'deferredRecip']):
629 time.sleep(self.timeInterval)
630 u.retrieveBmsearchId(self.updateData[queueid][
'msgid'])
631 u.replaceDSNRecip(self.updateData[queueid])
636 del self.updateData[queueid]
642 for badKey
in (
'table hash',
'warning',
'statistics'):
643 if badKey
in self.data[queueid]:
645 del self.data[queueid]
Definition: sqlQuery.py:1
Definition: requestToELK.py:1