#!/usr/bin/python3

import logging as log
from configobj import ConfigObj
import subprocess
import json
import os

"""
Runs a test against a rgw with the data cache enabled. A client must be
set in the config for this task. This client must be the same client
that is in the config for the `rgw` task.

In the `overrides` section `datacache` and `datacache` must be configured for
the `rgw` task and the ceph conf overrides must contain the below config
variables in the client section.

`s3cmd` must be added as an extra_package to the install task.

In the `workunit` task, `- rgw/run-datacache.sh` must be set for the client that
is in the config for the `rgw` task. The `RGW_DATACACHE_PATH` variable must be
set in the workunit's `env` and it must match the `datacache_path` given to the
`rgw` task in `overrides`.
Ex:
- install:
    extra_packages:
      deb: ['s3cmd']
      rpm: ['s3cmd']
- overrides:
    rgw:
      datacache: true
      datacache_path: /tmp/rgw_datacache
    install:
      extra_packages:
        deb: ['s3cmd']
        rpm: ['s3cmd']
    ceph:
      conf:
        client:
          rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/
          rgw d3n l1 datacache size: 10737417240
          rgw d3n l1 local datacache enabled: true
          rgw enable ops log: true
- rgw:
    client.0:
- workunit:
    clients:
      client.0:
      - rgw/run-datacache.sh
    env:
      RGW_DATACACHE_PATH: /tmp/rgw_datacache
    cleanup: true
"""

log.basicConfig(level=log.DEBUG)

""" Constants """
USER = 'rgw_datacache_user'
DISPLAY_NAME = 'DatacacheUser'
ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A'
SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
BUCKET_NAME = 'datacachebucket'
FILE_NAME = '7M.dat'
GET_FILE_NAME = '7M-get.dat'

def exec_cmd(cmd):
    log.debug("exec_cmd(%s)", cmd)
    try:
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out, err = proc.communicate()
        if proc.returncode == 0:
            log.info('command succeeded')
            if out is not None: log.info(out)
            return out
        else:
            raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode))
    except Exception as e:
        log.error('command failed')
        log.error(e)
        return False

def get_radosgw_endpoint():
    out = exec_cmd('sudo netstat -nltp | egrep "rados|valgr"')  # short for radosgw/valgrind
    x = out.decode('utf8').split(" ")
    port = [i for i in x if ':' in i][0].split(':')[1]
    log.info('radosgw port: %s' % port)
    proto = "http"
    hostname = '127.0.0.1'

    if port == '443':
        proto = "https"

    endpoint = hostname

    log.info("radosgw endpoint is: %s", endpoint)
    return endpoint, proto

def create_s3cmd_config(path, proto):
    """
    Creates a minimal config file for s3cmd
    """
    log.info("Creating s3cmd config...")

    use_https_config = "False"
    log.info("proto for s3cmd config is %s", proto)
    if proto == "https":
        use_https_config = "True"

    s3cmd_config = ConfigObj(
        indent_type='',
        infile={
            'default':
                {
                'host_bucket': 'no.way.in.hell',
                'use_https': use_https_config,
                },
            }
    )

    f = open(path, 'wb')
    s3cmd_config.write(f)
    f.close()
    log.info("s3cmd config written")

def get_cmd_output(cmd_out):
    out = cmd_out.decode('utf8')
    out = out.strip('\n')
    return out


def main():
    """
    execute the datacache test
    """
    # setup for test
    cache_dir = os.environ['RGW_DATACACHE_PATH']
    log.debug("datacache dir from config is: %s", cache_dir)

    out = exec_cmd('pwd')
    pwd = get_cmd_output(out)
    log.debug("pwd is: %s", pwd)

    endpoint, proto = get_radosgw_endpoint()

    # create 7M file to put
    outfile = pwd + '/' + FILE_NAME
    exec_cmd('dd if=/dev/urandom of=%s bs=1M count=7' % (outfile))

    # create user
    exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s'
            % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY))

    # create s3cmd config
    s3cmd_config_path = pwd + '/s3cfg'
    create_s3cmd_config(s3cmd_config_path, proto)

    # create a bucket
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s mb s3://%s'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME))

    # put an object in the bucket
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s put %s s3://%s'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, outfile, BUCKET_NAME))

    # get object from bucket
    get_file_path = pwd + '/' + GET_FILE_NAME
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s get s3://%s/%s %s --force'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME, FILE_NAME, get_file_path))

    # get info of object
    out = exec_cmd('radosgw-admin object stat --bucket=%s --object=%s' % (BUCKET_NAME, FILE_NAME))

    json_op = json.loads(out)
    cached_object_name = json_op['manifest']['prefix']
    log.debug("Cached object name is: %s", cached_object_name)
    stripe_size = json_op['manifest']['begin_iter']['stripe_size']
    log.debug("*** stripe_size = %s", stripe_size)
    stripe_max_size = json_op['manifest']['rules'][0]['val']['stripe_max_size']
    log.debug("*** stripe_max_size = %s", stripe_max_size)
    compression_type = 'none'
    if 'compression' in json_op and 'compression_type' in json_op['compression']:
        compression_type = json_op['compression']['compression_type']
    log.debug("*** compression_type = %s", compression_type)

    # check if the cache directory is empty or not
    out = exec_cmd('find %s -type f | wc -l' % (cache_dir))
    cache_dir_file_count = int(get_cmd_output(out))
    log.debug("*** cache_dir_file_count = %s", cache_dir_file_count)

    if stripe_max_size > 4194304:
        # ref: qa/suites/rgw/verify/striping$/stripe-greater-than-chunk.yaml : "rgw obj stripe size: 6291456"
        log.debug("*** NOTICE: Running under 'stripe-greater-than-chunk' permutation - skipping the RGW Datacache test")
        return
    elif compression_type != 'none':
        log.debug("*** NOTICE: Running under 'compression' enabled permutation - skipping the RGW Datacache test")
        return
    else:
        log.debug("*** NOTICE: RGW Datacache test will be performed")

    if cache_dir_file_count == 0:
        log.debug("*** ERROR: Cache directory is empty, RGW Datacache test FAILED")
        assert(cache_dir_file_count > 0)

    # list the files in the cache dir for troubleshooting
    out = exec_cmd('ls -ltr %s' % (cache_dir))
    # get name of cached object and check if it exists in the cache
    out = exec_cmd('ls -1tr %s | tail -1' % (cache_dir))
    cached_object_path = cache_dir + "/" + get_cmd_output(out)

    out = exec_cmd('ls -ltr %s' % (cache_dir))
    log.debug("Path of file in datacache is: %s", cached_object_path)
    out = exec_cmd("sha1sum %s  | awk '{ print $1 }'" % (cached_object_path))
    cached_object_sha1 = get_cmd_output(out)
    log.debug("*** SHA1 of file in datacache is: %s", cached_object_sha1)

    # check to see if the cached object is in Ceph
    out = exec_cmd('rados ls -p default.rgw.buckets.data')
    rados_ls_out = get_cmd_output(out)
    log.debug("rados ls output is: %s", rados_ls_out)

    out = exec_cmd("dd status=none if=%s of=/dev/stdout bs=1M skip=4 | sha1sum | awk '{ print $1 }'" % (outfile))
    org_object_sha1 = get_cmd_output(out)
    log.debug("*** SHA1 of cached part in original file is: %s", org_object_sha1)
    out = exec_cmd("dd status=none if=%s of=/dev/stdout bs=1M skip=4 | sha1sum | awk '{ print $1 }'" % (get_file_path))
    download_object_sha1 = get_cmd_output(out)
    log.debug("*** SHA1 of cached part in downloaded file is: %s", download_object_sha1)

    log.debug("*** CHECK: cache_dir_file_count: %s > 0  and  (cached_object_sha1: %s == org_object_sha1: %s  or  (stripe_size: %s < 4194304 and org_object_sha1: %s == download_object_sha1: %s)", cache_dir_file_count, cached_object_sha1, org_object_sha1, stripe_size, org_object_sha1, download_object_sha1)
    assert(cache_dir_file_count > 0  and  (cached_object_sha1 == org_object_sha1  or  (stripe_size < 4194304 and org_object_sha1 == download_object_sha1)))
    #                                     (cached_object_sha1 == org_object_sha1) test fails if "stripe_size" < 4MiB(4194304),
    #                                     in that case fall back to checking the sha1 of the downloaded file (https://tracker.ceph.com/issues/71387)

    log.debug("RGW Datacache test SUCCESS")

    # remove datacache dir
    #cmd = exec_cmd('rm -rf %s' % (cache_dir))
    #log.debug("RGW Datacache dir deleted")
    #^ commenting for future reference - the work unit will continue running tests and if the cache_dir is removed
    #  all the writes to cache will fail with errno 2 ENOENT No such file or directory.

main()
log.info("Completed Datacache tests")
