Amazon Kinesis Agent for Windows を使用して、Windows DHCP Server ログを即時に利用可能なメトリクスへ変換する

24,10/19/18,00:00:18,Database Cleanup Begin,,,,,0,6,,,,,,,,,0
18,10/19/18,00:00:18,Expired,192.168.1.251,,,,0,6,,,,,,,,,0
30,10/19/18,00:00:18,DNS Update Request,192.168.1.35,TEST-SERVER.domain.com,,,0,6,,,,,,,,,0
17,10/19/18,00:00:18,DNS record not deleted,192.168.5.35,,,,0,6,,,,,,,,,0
25,10/19/18,00:00:18,0 leases expired and 5 leases deleted,,,,,0,6,,,,,,,,,0
32,10/19/18,00:00:18,DNS Update Successful,192.168.1.35,TEST-SERVER.domain.com,,,0,6,,,,,,,,,0
14,10/19/18,00:00:19,Scope Full,192.168.3.10,,,,0,6,,,,,,,,,0
11,10/19/18,00:00:20,Renew,192.168.2.105,,00AABBCCDDEE,,1584371322,0,,,,0x506F6C79636F6D2D53504950333335,Polycom,,,,0
36,10/19/18,00:00:25,Packet dropped because of Client ID hash mismatch or standby server.,192.168.1.100,,EEDDCCBBAA00,,0,6,,,,,,,,,0

# Get existing objects
$s3BucketName = '<s3-bucket-name>'
$firehoseDeliveryStreamName = '<delivery-stream-name>'
$roleName = '<role-name>'

$s3Bucket = Get-S3Bucket -BucketName $s3BucketName
$firehoseDeliveryStream = Get-KINFDeliveryStream -DeliveryStreamName $firehoseDeliveryStreamName
$iamRole = Get-IAMRole -RoleName $roleName

# Enable S3 Bucket archival to Glacier
$s3BucketLifecycleRuleId = 'Archive to Glacier after 14 days'
$s3BucketLifecycleRuleTransition = New-Object Amazon.S3.Model.LifecycleTransition
$s3BucketLifecycleRuleTransition.Days = 14
$s3BucketLifecycleRuleTransition.StorageClass = 'GLACIER'

$s3BucketLifecycleRule = New-Object Amazon.S3.Model.LifecycleRule
$s3BucketLifecycleRule.Id = $s3BucketLifecycleRuleId
$s3BucketLifecycleRule.Status = 'Enabled'
$s3BucketLifecycleRule.Transition = $s3BucketLifecycleRuleTransition
$s3BucketLifecycleRule.Prefix = $null

Write-S3LifecycleConfiguration –BucketName $s3Bucket.BucketName -Configuration_Rule $s3BucketLifecycleRule

# Enable CW Logging options including creation of a log group and a log stream 
$logGroupName = "/aws/kinesisfirehose/$firehoseDeliveryStreamName"
$logStreamName = 'S3Delivery'

# Create CloudWatch LogGroup and LogStream
New-CWLLogGroup -LogGroupName $logGroupName
New-CWLLogStream -LogGroupName $logGroupName -LogStreamName $logStreamName

# Define Kinesis Firehose Logging Options
$loggingOptions = New-Object Amazon.KinesisFirehose.Model.CloudWatchLoggingOptions
$loggingOptions.Enabled = $true
$loggingOptions.LogGroupName = $logGroupName
$loggingOptions.LogStreamName = $logStreamName

# Define Buffering hints object for traffic between Delivery Stream and S3
$bufferingHints = New-Object Amazon.KinesisFirehose.Model.BufferingHints
$bufferingHints.IntervalInSeconds = 60
$bufferingHints.SizeInMBs = 128

# Define Kinesis Firehose S3 Destination Update
$s3Destination = New-Object Amazon.KinesisFirehose.Model.ExtendedS3DestinationUpdate
$s3Destination.BucketARN = 'arn:aws:s3:::{0}' -f $s3Bucket.BucketName
$s3Destination.RoleARN = $iamRole.Arn
$s3Destination.CompressionFormat = 'GZIP'
$s3Destination.BufferingHints = $bufferingHints
$s3Destination.Prefix = 'DHCPServerLogs'
$s3Destination.CloudWatchLoggingOptions = $loggingOptions

# Update the Kinesis Firehose Delivery Stream
$kinfdUpdateParams = @{
    CurrentDeliveryStreamVersionId   = $firehoseDeliveryStream.VersionId
    DeliveryStreamName               = $firehoseDeliveryStreamName
    DestinationId                    = $firehoseDeliveryStream.Destinations.DestinationId
    ExtendedS3DestinationUpdate      = $s3Destination
}
Update-KINFDestination @kinfdUpdateParams

# Create the SNS topic.作成したトピックの ARN が返されます。
$snsTopicName = 'DHCPLogLambdaNotifier'
$topicArn = New-SNSTopic -Name $snsTopicName

# Give S3 permissions to send notifications to the Topic
$topicPolicy = @"
{
    "Version": "2008-10-17",
    "Id": "__default_policy_ID",
    "Statement": [
        {
            "Sid": "S3Publish",
            "Effect": "Allow",
            "Principal": {
                "AWS": "*"
            },
            "Action": [
                "SNS:Publish"
            ],
            "Resource": "$topicArn",
            "Condition": {
                "StringEquals": {
                    "aws:SourceArn": "arn:aws:s3:::$s3BucketName"
                }
            }
        }
    ]
}
"@
Set-SNSTopicAttribute -TopicArn $topicArn -AttributeName 'Policy' -AttributeValue $topicPolicy

# Configure S3 Bucket SNS Topic Event Notification
$s3BucketTopicConfigurationFilterRule = New-Object Amazon.S3.Model.FilterRule
$s3BucketTopicConfigurationFilterRule.Name = 'Prefix'
$s3BucketTopicConfigurationFilterRule.Value = 'DHCPServerLogs/'

$s3BucketTopicConfigurationS3KeyFilter = New-Object Amazon.S3.Model.S3KeyFilter
$s3BucketTopicConfigurationS3KeyFilter.FilterRules = $s3BucketTopicConfigurationFilterRule

$s3BucketTopicConfigurationFilter = New-Object Amazon.S3.Model.Filter
$s3BucketTopicConfigurationFilter.S3KeyFilter = $s3BucketTopicConfigurationS3KeyFilter

$s3BucketTopicConfiguration = New-Object Amazon.S3.Model.TopicConfiguration
$s3BucketTopicConfiguration.Id = 'NotifySNS'
$s3BucketTopicConfiguration.Topic = $topicArn
$s3BucketTopicConfiguration.Filter = $s3BucketTopicConfigurationFilter
$s3BucketTopicConfiguration.Events = New-Object Amazon.S3.EventType 's3:ObjectCreated:*'
Write-S3BucketNotification -BucketName $s3BucketName -TopicConfiguration $s3BucketTopicConfiguration

# Create IAM policy and add AssumeRole policy for Lambda
$lambdaPolicy = @"
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "CloudWatch",
            "Effect": "Allow",
            "Action": [
                  "cloudwatch:PutMetricData",
                  "cloudwatch:GetMetricData",
                  "cloudwatch:GetMetricStatistics",
                  "cloudwatch:ListMetrics"
            ],
            "Resource": "*"
        },
        {
            "Sid": "S3",
            "Effect": "Allow",
            "Action": [
                  "s3:ListBucket",
                  "s3:GetObject",
                  "s3:GetBucketLocation",
                  "s3:GetBucketNotification"
            ],
            "Resource": [
                  "arn:aws:s3:::$s3BucketName",
                  "arn:aws:s3:::$s3BucketName/*"
            ]
        },
        {
            "Sid": "Log",
            "Effect": "Allow",
            "Action": [
                  "logs:CreateLogStream",
                  "logs:PutLogEvents"
            ],
            "Resource": [
                  "arn:aws:logs:*:*:*"
            ]
        }
    ]
}
"@

$iamRoleName = 'LambdaDHCPLogProcessor'
New-IAMPolicy -PolicyName $iamRoleName -PolicyDocument $lambdaPolicy
$assumeRolePolicy = @"
{
    "Version": "2012-10-17",
    "Statement": [
    {
        "Sid": "LambdaAssumeRole",
        "Effect": "Allow",
        "Principal": {
            "Service": "lambda.amazonaws.com"
        },
        "Action": [
            "sts:AssumeRole"
        ]
    }]
}
"@
$lambdaIamRole = New-IAMRole -RoleName $iamRoleName -AssumeRolePolicyDocument $assumeRolePolicy
$iamPolicy = Get-IAMPolicies | Where-Object {$_.PolicyName -eq $iamRoleName}
Register-IAMRolePolicy -RoleName $iamRoleName -PolicyArn $iamPolicy.Arn

# Create the Lambda Function
$lambdaFunctionParams = @{
        Description = 'For DHCP lease refusal metric posting'
        FunctionName = 'LeaseRefusalMetrics'
        ZipFilename = '.\LeaseRefusalMetrics.zip'
        Handler = 'LeaseRefusalMetrics.lambda_handler'
        Role = $lambdaIamRole.Arn
        Runtime = 'python3.6'
}
$lambdaFunction = Publish-LMFunction @lambdaFunctionParams

# Subscribe the Lambda Function to the SNS topic
$snsSubscriptionArn = Connect-SNSNotification -TopicARN $topicArn -Protocol Lambda -Endpoint $lambdaFunction.FunctionArn

# Add permission to the Lambda Function's policy so SNS can invoke it
Add-LMPermission -FunctionName $lambdaFunctionParams.FunctionName -Action "lambda:Invoke" -Principal sns.amazonaws.com -SourceArn $topicArn -StatementId (Get-Random)

from __future__ import print_function
import json
import urllib.parse
from io import BytesIO
from gzip import GzipFile
import datetime
from datetime import timedelta
import re
import boto3

s3Client = boto3.client('s3')
cwClient = boto3.client('cloudwatch')

def lambda_handler(event, context):
    print(json.dumps(event))

    s3message = json.loads(event['Records'][0]['Sns']['Message'])
    print(json.dumps(s3message))

    bucket = s3message['Records'][0]['s3']['bucket']['name']
    key = urllib.parse.unquote_plus(s3message['Records'][0]['s3']['object']['key'], encoding='utf-8')

    # Decompress the S3 object and read data in as byte stream
    try:
        response = s3Client.get_object(Bucket=bucket, Key=key)
        bytestream = BytesIO(response['Body'].read())
        s3object_text = GzipFile(None, 'rb', fileobj=bytestream).read().decode('utf-8')
    except Exception as e:
        print(e)
        print('Error getting object {} from bucket {}.Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
        raise e


    # Split ByteStream into individual records and search for event 14 (scope exhausted)
    # When Event 14 is found, break apart each individual CSV record to parse out values
    # for metric dimensions: dhcpFailoverRelationshipName, dhcpScopeId.Finally, post a metric to CloudWatch.
    # Sample record: "DHCP-FailoverRelationshipName-01:::14,11/06/18,03:04:00,Scope Full,192.168.243.0,,,,0,6,,,,,,,,,0"

    records = s3object_text.splitlines()
    for record in records:
        recordValues = ""
        hostTemp = ""
        temp = ""
        date = ""
        time = ""
        dhcpFailoverRelationshipNameTemp = ""
        dhcpFailoverRelationshipName = ""
        dhcpHostName = ""
        dhcpScopeId = ""

        if re.match(r'^DHCP-.*:::14,', record, re.IGNORECASE):
            print("DHCP Full Scope Event: " + record)
            recordValues = record.split(',')
            hostTemp = recordValues[0]
            dhcpScopeId = str(recordValues[4])

            eventDate = str(recordValues[1]).replace("/","-")
            eventTime = recordValues[2]
            strDateTime = eventDate + " " + eventTime
            eventDTime = datetime.datetime.strptime(strDateTime, "%m-%d-%y %H:%M:%S")

            temp = hostTemp.split(':::')
            dhcpHostName = temp[0]
            dhcpFailoverRelationshipNameTemp = dhcpHostName.split('-')
            dhcpFailoverRelationshipName = dhcpFailoverRelationshipNameTemp[1]

            response = cwClient.put_metric_data(
                Namespace='DHCPService',
                MetricData=[
                    {
                        'MetricName': 'ScopeFullLeaseRefusal',
                        'Dimensions': [
                            {
                                'Name': 'DHCPFailoverRelationshipName',
                                'Value': dhcpFailoverRelationshipName
                            },
                            {
                                'Name': 'DHCPScopeId',
                                'Value': dhcpScopeId
                            }
                        ],
                        'Timestamp': eventDTime,
                        'Value': 1,
                        'Unit': 'Count',
                        'StorageResolution': 60
                    },
                    {
                        'MetricName': 'ScopeFullLeaseRefusal',
                        'Dimensions': [
                            {
                                'Name': 'DHCPFailoverRelationshipName',
                                'Value': dhcpFailoverRelationshipName
                            }
                        ],
                        'Timestamp': strDateTime,
                        'Value': 1,
                        'Unit': 'Count',
                        'StorageResolution': 60
                    },
                    {
                        'MetricName': 'AggregateLeaseRefusal',
                        'Dimensions': [
                            {
                                'Name': 'ScopeFullLeaseRefusal',
                                'Value': 'ScopeFullLeaseRefusal'
                            }
                        ],
                        'Timestamp': strDateTime,
                        'Value': 1,
                        'Unit': 'Count',
                        'StorageResolution': 60
                    }
                ]
            )

            print("CW.Put_Metric Response: ", response)

{
    "Sources": [
        {
            "Id": "DHCPServerLog",
            "SourceType": "DirectorySource",
            "Directory": "C:\\Windows\\System32\\dhcp",
            "FileNameFilter": "Dhcp*SrvLog-*.log", <--------------- Only capture specific file names
            "InitialPosition": "Bookmark",
            "RecordParser": "SingleLine" <------------------------- Indicates that each distinct record is line delimited
        }
    ],
    "Pipes": [
        {
            "Id": "DHCPServerLog2Firehose",
            "SourceRef": "DHCPServerLog",
            "SinkRef": "DHCPServerLogsToFirehose",
            "Type": "RegexFilterPipe",
            "FilterPattern": "^\\d{2},.*" <------------------------ Indicates to only collect records that begin with two 
        }                                                           digits followed by a comma.Recall our log file has a
    ],                                                              header which we don't need to collect.
    "Sinks": [
        {
            "Id": "DHCPServerLogsToFirehose",
            "SinkType": "KinesisFirehose",
            "StreamName": "DHCPServerLogs",
            "TextDecoration": "{ComputerName}:::{_record}" <------- Decorate each log record by pre-pending the hostname 
        }                                                           and the following custom delimiter ':::'
    ]
}