Week 06: 测试驱动开发与云原生执行环境

# 安装 Molecule 及 Docker 驱动
pip install molecule molecule-plugins[docker]

# 在现有 Role 中初始化 Molecule
cd roles/nginx
molecule init scenario --driver-name docker

# 生成的目录结构
roles/nginx/
├── molecule/
│   └── default/
│       ├── converge.yml      # 执行 Role 的 Playbook
│       ├── molecule.yml      # 场景配置
│       ├── prepare.yml       # 环境预配置 (可选)
│       ├── verify.yml        # 验证 Playbook
│       └── cleanup.yml       # 清理 Playbook (可选)
├── tasks/
├── defaults/
└── ...

# molecule/default/molecule.yml
---
dependency:
  name: galaxy
  options:
    requirements-file: requirements.yml
    role-file: requirements.yml

driver:
  name: docker

platforms:
  # 多平台矩阵测试
  - name: centos-stream-9
    image: quay.io/centos/centos:stream9
    pre_build_image: true
    command: /usr/sbin/init
    privileged: true
    volumes:
      - /sys/fs/cgroup:/sys/fs/cgroup:rw
    cgroupns_mode: host
    
  - name: ubuntu2204
    image: geerlingguy/docker-ubuntu2204-ansible
    pre_build_image: true
    
  - name: debian12
    image: geerlingguy/docker-debian12-ansible
    pre_build_image: true

provisioner:
  name: ansible
  playbooks:
    prepare: prepare.yml
    converge: converge.yml
    verify: verify.yml
  inventory:
    host_vars:
      centos-stream-9:
        ansible_python_interpreter: /usr/bin/python3
      ubuntu2204:
        ansible_python_interpreter: /usr/bin/python3
  config_options:
    defaults:
      callbacks_enabled: profile_tasks, timer
      gathering: smart
      fact_caching: jsonfile
      fact_caching_connection: /tmp/molecule_facts
    ssh_connection:
      pipelining: true

verifier:
  name: ansible
  # 或使用 testinfra
  # name: testinfra
  # options:
  #   v: true

scenario:
  name: default
  test_sequence:
    - dependency
    - cleanup
    - destroy
    - syntax
    - create
    - prepare
    - converge
    - idempotence
    - side_effect
    - verify
    - cleanup
    - destroy

# molecule/default/converge.yml
---
- name: Converge
  hosts: all
  become: true
  
  vars:
    # 测试专用变量覆盖
    nginx_http_port: 8080
    nginx_worker_processes: 2
    nginx_vhosts:
      - server_name: test.local
        root: /var/www/test
  
  pre_tasks:
    - name: Update apt cache (Debian)
      apt:
        update_cache: yes
        cache_valid_time: 3600
      when: ansible_os_family == 'Debian'
  
  roles:
    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"

# molecule/default/verify.yml
---
- name: Verify
  hosts: all
  become: true
  gather_facts: true
  
  tasks:
    - name: Gather package facts
      package_facts:
        manager: auto
      
    - name: Assert nginx is installed
      assert:
        that:
          - "'nginx' in ansible_facts.packages"
        fail_msg: "NGINX package is not installed"
        success_msg: "NGINX package is installed"
    
    - name: Gather service facts
      service_facts:
    
    - name: Assert nginx service is running
      assert:
        that:
          - ansible_facts.services['nginx.service'].state == 'running'
          - ansible_facts.services['nginx.service'].status == 'enabled'
        fail_msg: "NGINX service is not running or enabled"
    
    - name: Check nginx is listening on port 8080
      wait_for:
        port: 8080
        state: started
        timeout: 10
    
    - name: Verify nginx responds to HTTP request
      uri:
        url: http://localhost:8080/
        status_code: [200, 301, 302, 403, 404]
      register: http_result
    
    - name: Assert HTTP response is successful
      assert:
        that:
          - http_result.status in [200, 301, 302, 403, 404]
    
    - name: Check configuration file syntax
      command: nginx -t
      register: nginx_test
      changed_when: false
    
    - name: Assert configuration is valid
      assert:
        that:
          - nginx_test.rc == 0

# molecule/default/tests/test_default.py
import pytest


def test_nginx_package_installed(host):
    """验证 NGINX 包已安装"""
    pkg = host.package("nginx")
    assert pkg.is_installed


def test_nginx_service_running(host):
    """验证 NGINX 服务正在运行"""
    svc = host.service("nginx")
    assert svc.is_running
    assert svc.is_enabled


def test_nginx_listening_on_port(host):
    """验证 NGINX 监听指定端口"""
    socket = host.socket("tcp://0.0.0.0:8080")
    assert socket.is_listening


def test_nginx_config_valid(host):
    """验证 NGINX 配置有效"""
    cmd = host.run("nginx -t")
    assert cmd.rc == 0
    assert "syntax is ok" in cmd.stderr


def test_nginx_config_file_exists(host):
    """验证配置文件存在且权限正确"""
    cfg = host.file("/etc/nginx/nginx.conf")
    assert cfg.exists
    assert cfg.is_file
    assert cfg.user == "root"
    assert cfg.group == "root"
    assert cfg.mode == 0o644


def test_vhost_config_exists(host):
    """验证虚拟主机配置存在"""
    vhost = host.file("/etc/nginx/sites-enabled/test.local.conf")
    assert vhost.exists
    assert vhost.is_symlink


@pytest.mark.parametrize("path,owner,mode", [
    ("/var/www/test", "www-data", 0o755),
    ("/var/log/nginx", "www-data", 0o755),
])
def test_directories_exist(host, path, owner, mode):
    """参数化测试: 验证目录存在且权限正确"""
    d = host.file(path)
    assert d.exists
    assert d.is_directory
    # 仅在 Debian 系列检查 owner
    if host.system_info.distribution in ['ubuntu', 'debian']:
        assert d.user == owner
    assert d.mode == mode

# Molecule 内部逻辑伪代码
def test_idempotence():
    result = run_playbook("converge.yml")
    if any(task.status == "changed" for task in result.tasks):
        fail("Idempotence test failed - tasks reported changed on second run")

# 创建额外场景
molecule init scenario --scenario-name ha --driver-name docker

# 目录结构
molecule/
├── default/              # 基础功能测试
│   └── molecule.yml
├── ha/                   # 高可用场景测试
│   └── molecule.yml
└── upgrade/              # 升级场景测试
    └── molecule.yml

# 运行指定场景
molecule test --scenario-name ha

# 运行所有场景
molecule test --all

# execution-environment.yml
---
version: 3

# 基础镜像 (支持 UBI, Alpine, 自定义)
images:
  base_image:
    name: quay.io/ansible/ansible-runner:latest
  # 或使用 EE 基础镜像
  # base_image:
  #   name: quay.io/ansible/awx-ee:latest

options:
  package_manager_path: /usr/bin/microdnf
  # 或 /usr/bin/dnf, /usr/bin/apt-get

dependencies:
  # Python 依赖
  python: requirements.txt
  
  # 系统包依赖 (bindep 格式)
  system: bindep.txt
  
  # Ansible Collections
  galaxy: requirements.yml

additional_build_files:
  # 添加自定义文件到构建上下文
  - src: files/custom-ca.crt
    dest: certs

additional_build_steps:
  # 在 galaxy 安装后执行
  prepend_galaxy:
    - RUN echo "Before galaxy install"
  
  # 在所有依赖安装后执行
  append_final:
    - COPY _build/certs/custom-ca.crt /etc/pki/ca-trust/source/anchors/
    - RUN update-ca-trust
    - RUN chmod +x /usr/local/bin/*

# AWS SDK
boto3>=1.26.0
botocore>=1.29.0

# Kubernetes
kubernetes>=25.0.0

# Database clients
psycopg2-binary>=2.9.0
pymysql>=1.0.0

# Cryptography
cryptography>=40.0.0

# HTTP clients
requests>=2.28.0
urllib3>=1.26.0

# JMESPath for json_query
jmespath>=1.0.0

# 编译工具
gcc [compile platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]
python3-devel [compile platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]
libffi-devel [compile platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]

# SSL 支持
openssl-devel [platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]
libssl-dev [platform:dpkg]

# Git (用于 git+ 格式的 requirements)
git [platform:centos-8 platform:centos-stream-8 platform:centos-stream-9 platform:dpkg]

# PostgreSQL 客户端库
postgresql-devel [platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]
libpq-dev [platform:dpkg]

# Kerberos (用于 Windows 模块)
krb5-devel [platform:centos-8 platform:centos-stream-8 platform:centos-stream-9]
libkrb5-dev [platform:dpkg]

---
collections:
  # AWS
  - name: amazon.aws
    version: ">=6.0.0"
  
  # Kubernetes
  - name: kubernetes.core
    version: ">=2.4.0"
  
  # 通用工具
  - name: ansible.utils
    version: ">=2.10.0"
  - name: community.general
    version: ">=7.0.0"
  
  # 数据库
  - name: community.postgresql
    version: ">=3.0.0"
  - name: community.mysql
    version: ">=3.0.0"
  
  # 网络自动化
  - name: ansible.netcommon
    version: ">=5.0.0"
  
  # 私有 Collection
  - name: my_org.infra_utils
    source: https://galaxy.example.com/
    version: ">=1.2.0"

# 安装 ansible-builder
pip install ansible-builder

# 构建 EE 镜像
ansible-builder build \
  --tag my-org-ee:1.0.0 \
  --container-runtime docker \
  --verbosity 3

# 构建输出结构
context/
├── _build/
│   ├── requirements.txt
│   ├── requirements.yml
│   └── bindep.txt
└── Containerfile  # 或 Dockerfile

ARG EE_BASE_IMAGE=quay.io/ansible/ansible-runner:latest
ARG PYCMD="/usr/bin/python3"
ARG PKGMGR="/usr/bin/microdnf"
ARG PYPKG="python3"
ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS=""
ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS=""
ARG ANSIBLE_INSTALL_REFS="ansible-core"

FROM $EE_BASE_IMAGE as galaxy
ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS
ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS
COPY _build/requirements.yml /tmp/requirements.yml
RUN ansible-galaxy role install $ANSIBLE_GALAXY_CLI_ROLE_OPTS -r /tmp/requirements.yml --roles-path /usr/share/ansible/roles
RUN ansible-galaxy collection install $ANSIBLE_GALAXY_CLI_COLLECTION_OPTS -r /tmp/requirements.yml --collections-path /usr/share/ansible/collections

FROM $EE_BASE_IMAGE as builder
# ... bindep 和 pip 安装 ...

FROM $EE_BASE_IMAGE as final
COPY --from=galaxy /usr/share/ansible /usr/share/ansible
COPY --from=builder /output/ /output/
RUN /output/install-from-bindep && rm -rf /output/wheels

# 优化后的配置
additional_build_steps:
  append_final:
    - RUN pip cache purge
    - RUN microdnf clean all
    - RUN rm -rf /root/.cache /tmp/*

# 安装 ansible-navigator
pip install ansible-navigator

# 使用 EE 执行 Playbook
ansible-navigator run site.yml \
  --execution-environment-image my-org-ee:1.0.0 \
  --mode stdout

# 交互式 TUI 模式
ansible-navigator run site.yml \
  --execution-environment-image my-org-ee:1.0.0 \
  --mode interactive

# ansible-navigator.yml
---
ansible-navigator:
  execution-environment:
    container-engine: docker
    enabled: true
    image: my-org-ee:1.0.0
    pull:
      policy: missing
    volume-mounts:
      - src: ~/.ssh
        dest: /home/runner/.ssh
        options: ro
      - src: /etc/ssl/certs
        dest: /etc/ssl/certs
        options: ro
    environment-variables:
      set:
        AWS_PROFILE: production
        
  logging:
    level: debug
    file: /tmp/ansible-navigator.log
    
  playbook-artifact:
    enable: true
    save-as: "{playbook_name}-artifact-{time_stamp}.json"

#!/usr/bin/python
# -*- coding: utf-8 -*-

# GNU General Public License v3.0+
# ANSIBLE METADATA (可选但推荐)

DOCUMENTATION = r'''
---
module: my_api_client

short_description: Interact with internal REST API

version_added: "1.0.0"

description:
    - This module provides an interface to the internal REST API.
    - It supports CRUD operations on resources.

options:
    endpoint:
        description:
            - The API endpoint URL.
        required: true
        type: str
    resource_id:
        description:
            - The ID of the resource to manage.
        required: false
        type: str
    state:
        description:
            - Desired state of the resource.
        required: false
        type: str
        choices: ['present', 'absent']
        default: 'present'
    data:
        description:
            - Resource data as a dictionary.
        required: false
        type: dict

author:
    - Your Name (@github_handle)
'''

EXAMPLES = r'''
- name: Create a resource
  my_api_client:
    endpoint: https://api.example.com/v1
    state: present
    data:
      name: my-resource
      type: compute

- name: Delete a resource
  my_api_client:
    endpoint: https://api.example.com/v1
    resource_id: res-12345
    state: absent
'''

RETURN = r'''
resource:
    description: The created or updated resource object.
    type: dict
    returned: when state is present
    sample:
        id: res-12345
        name: my-resource
        status: active
changed:
    description: Whether any changes were made.
    type: bool
    returned: always
'''

from ansible.module_utils.basic import AnsibleModule
import json

try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False


def get_resource(module, endpoint, resource_id, headers):
    """获取资源当前状态"""
    url = f"{endpoint}/resources/{resource_id}"
    try:
        response = requests.get(url, headers=headers, timeout=30)
        if response.status_code == 404:
            return None
        response.raise_for_status()
        return response.json()
    except requests.RequestException as e:
        module.fail_json(msg=f"Failed to get resource: {str(e)}")


def create_resource(module, endpoint, data, headers):
    """创建资源"""
    url = f"{endpoint}/resources"
    try:
        response = requests.post(url, json=data, headers=headers, timeout=30)
        response.raise_for_status()
        return response.json()
    except requests.RequestException as e:
        module.fail_json(msg=f"Failed to create resource: {str(e)}")


def update_resource(module, endpoint, resource_id, data, headers):
    """更新资源"""
    url = f"{endpoint}/resources/{resource_id}"
    try:
        response = requests.put(url, json=data, headers=headers, timeout=30)
        response.raise_for_status()
        return response.json()
    except requests.RequestException as e:
        module.fail_json(msg=f"Failed to update resource: {str(e)}")


def delete_resource(module, endpoint, resource_id, headers):
    """删除资源"""
    url = f"{endpoint}/resources/{resource_id}"
    try:
        response = requests.delete(url, headers=headers, timeout=30)
        if response.status_code == 404:
            return False  # 已不存在
        response.raise_for_status()
        return True
    except requests.RequestException as e:
        module.fail_json(msg=f"Failed to delete resource: {str(e)}")


def resources_equal(current, desired):
    """比较资源是否相同 (用于幂等性判断)"""
    # 忽略系统生成的字段
    ignore_keys = {'id', 'created_at', 'updated_at', 'status'}
    current_filtered = {k: v for k, v in current.items() if k not in ignore_keys}
    desired_filtered = {k: v for k, v in desired.items() if k not in ignore_keys}
    return current_filtered == desired_filtered


def run_module():
    # 定义模块参数
    module_args = dict(
        endpoint=dict(type='str', required=True),
        resource_id=dict(type='str', required=False, default=None),
        state=dict(type='str', choices=['present', 'absent'], default='present'),
        data=dict(type='dict', required=False, default=None),
        api_token=dict(type='str', required=False, no_log=True),
    )

    # 初始化返回结果
    result = dict(
        changed=False,
        resource=None,
        diff=dict(before=None, after=None),
    )

    # 创建模块实例
    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True,
        required_if=[
            ('state', 'present', ['data']),
            ('state', 'absent', ['resource_id']),
        ],
    )

    # 检查依赖
    if not HAS_REQUESTS:
        module.fail_json(msg="The 'requests' library is required. Install with: pip install requests")

    # 提取参数
    endpoint = module.params['endpoint'].rstrip('/')
    resource_id = module.params['resource_id']
    state = module.params['state']
    data = module.params['data']
    api_token = module.params['api_token']

    headers = {'Content-Type': 'application/json'}
    if api_token:
        headers['Authorization'] = f'Bearer {api_token}'

    # 处理 state: absent
    if state == 'absent':
        current = get_resource(module, endpoint, resource_id, headers)
        
        if current is None:
            # 资源已不存在, 无需操作
            result['changed'] = False
        else:
            result['diff']['before'] = current
            result['diff']['after'] = None
            
            if module.check_mode:
                result['changed'] = True
            else:
                delete_resource(module, endpoint, resource_id, headers)
                result['changed'] = True
        
        module.exit_json(**result)

    # 处理 state: present
    if state == 'present':
        current = None
        if resource_id:
            current = get_resource(module, endpoint, resource_id, headers)

        if current is None:
            # 资源不存在, 需要创建
            result['diff']['before'] = None
            result['diff']['after'] = data
            
            if module.check_mode:
                result['changed'] = True
                result['resource'] = data
            else:
                created = create_resource(module, endpoint, data, headers)
                result['changed'] = True
                result['resource'] = created
        else:
            # 资源存在, 检查是否需要更新
            if resources_equal(current, data):
                # 无需变更
                result['changed'] = False
                result['resource'] = current
            else:
                # 需要更新
                result['diff']['before'] = current
                result['diff']['after'] = data
                
                if module.check_mode:
                    result['changed'] = True
                    result['resource'] = {**current, **data}
                else:
                    updated = update_resource(module, endpoint, resource_id, data, headers)
                    result['changed'] = True
                    result['resource'] = updated
        
        module.exit_json(**result)


def main():
    run_module()


if __name__ == '__main__':
    main()

# 模式 1: 提前返回
if module.check_mode:
    result['changed'] = would_change
    module.exit_json(**result)

# 模式 2: 条件执行
if not module.check_mode:
    actual_result = perform_action()
    result['resource'] = actual_result
result['changed'] = True

result = dict(
    changed=True,
    diff=dict(
        before="old_content\n",
        after="new_content\n",
        # 或者使用结构化格式
        before_header=f"/etc/app/config.ini (content)",
        after_header=f"/etc/app/config.ini (content)",
    )
)

# 项目级别 (推荐用于自定义模块)
playbooks/
├── library/
│   └── my_api_client.py
└── site.yml

# Role 级别
roles/myapp/
├── library/
│   └── my_api_client.py
└── tasks/

# Collection 级别
my_org/infra_utils/
└── plugins/
    └── modules/
        └── my_api_client.py

# .gitlab-ci.yml
---
stages:
  - lint
  - test
  - build
  - deploy

variables:
  ANSIBLE_FORCE_COLOR: "true"
  PY_COLORS: "1"
  
image: python:3.11-slim

cache:
  paths:
    - .cache/pip
    - .tox

before_script:
  - pip install --cache-dir .cache/pip ansible ansible-lint molecule molecule-plugins[docker]

# 代码检查
lint:
  stage: lint
  script:
    - ansible --version
    - ansible-lint --version
    - ansible-playbook playbooks/*.yml --syntax-check
    - ansible-lint playbooks/ roles/
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# 单元测试 (使用 Docker-in-Docker)
molecule_test:
  stage: test
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip install ansible molecule molecule-plugins[docker]
  script:
    - cd roles/nginx
    - molecule test
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - roles/**/*

# 构建 EE 镜像
build_ee:
  stage: build
  image: docker:24-dind
  services:
    - docker:24-dind
  variables:
    DOCKER_HOST: tcp://docker:2375
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip install ansible-builder
  script:
    - ansible-builder build --tag $CI_REGISTRY_IMAGE/ee:$CI_COMMIT_SHORT_SHA
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker push $CI_REGISTRY_IMAGE/ee:$CI_COMMIT_SHORT_SHA
  only:
    - tags

# 部署到 Staging
deploy_staging:
  stage: deploy
  image: $CI_REGISTRY_IMAGE/ee:$CI_COMMIT_SHORT_SHA
  script:
    - ansible-playbook -i inventory/staging playbooks/site.yml --diff
  environment:
    name: staging
  rules:
    - if: $CI_COMMIT_BRANCH == "develop"
  when: manual

# 部署到 Production
deploy_production:
  stage: deploy
  image: $CI_REGISTRY_IMAGE/ee:$CI_COMMIT_SHORT_SHA
  script:
    - ansible-playbook -i inventory/production playbooks/site.yml --diff --check
  environment:
    name: production
  rules:
    - if: $CI_COMMIT_TAG
  when: manual

# .github/workflows/ansible.yml
---
name: Ansible CI

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'
      
      - name: Install dependencies
        run: pip install ansible ansible-lint yamllint
      
      - name: Run yamllint
        run: yamllint .
      
      - name: Run ansible-lint
        run: ansible-lint playbooks/ roles/
      
      - name: Syntax check
        run: ansible-playbook playbooks/*.yml --syntax-check

  molecule:
    needs: lint
    runs-on: ubuntu-latest
    strategy:
      matrix:
        role:
          - nginx
          - postgresql
          - common-security
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      
      - name: Install dependencies
        run: pip install ansible molecule molecule-plugins[docker]
      
      - name: Run Molecule tests
        working-directory: roles/${{ matrix.role }}
        run: molecule test

  security:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: 'fs'
          scan-ref: '.'
          format: 'sarif'
          output: 'trivy-results.sarif'
      
      - name: Upload Trivy scan results
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: 'trivy-results.sarif'

# molecule/default/molecule.yml (添加)
provisioner:
  name: ansible
  config_options:
    defaults:
      callbacks_enabled: profile_tasks, timer, junit
      callback_whitelist: profile_tasks, timer, junit
      
verifier:
  name: testinfra
  options:
    junit-xml: molecule_junit.xml
    cov: roles/nginx
    cov-report: html