Merge remote-tracking branch 'origin/master'

Author: lhhyung

.github/workflows/dispatch_build_dev.yaml

@@ -30,51 +30,51 @@ jobs:
           author_name: Github Action Slack
 
   docker:
-      if: github.repository_owner == 'cloudforet-io'
-      needs: versioning
-      runs-on: ubuntu-latest
-      env:
-        VERSION: ${{ needs.versioning.outputs.version }}
-      steps:
-        - name: Checkout
-          uses: actions/checkout@v3
-          with:
-            token: ${{ secrets.PAT_TOKEN }}
-  
-        - name: get service name
-          run: |
-            echo "SERVICE=$(echo ${{ github.repository }} | cut -d '/' -f2)" >> $GITHUB_ENV
-            
-        - name: Set up QEMU
-          uses: docker/setup-qemu-action@v2
-  
-        - name: Set up Docker Buildx
-          uses: docker/setup-buildx-action@v2
-  
-        - name: Login to Docker Hub
-          uses: docker/login-action@v2
-          with:
-            username: ${{ secrets.DOCKER_USERNAME }}
-            password: ${{ secrets.DOCKER_PASSWORD }}
-            
-        - name: Build and push to pyengine
-          uses: docker/build-push-action@v4
-          with:
-            context: .
-            push: true 
-            tags: pyengine/${{ env.SERVICE }}:${{ env.VERSION }}
-            
-        - name: Notice when job fails
-          if: failure()
-          uses: 8398a7/action-slack@v3.2.0
-          with:
-            status: ${{job.status}}
-            fields: repo,workflow,job
-            author_name: Github Action Slack
+    if: github.repository_owner == 'cloudforet-io'
+    needs: versioning
+    runs-on: ubuntu-latest
+    env:
+      VERSION: ${{ needs.versioning.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PAT_TOKEN }}
+
+      - name: get service name
+        run: |
+          echo "SERVICE=$(echo ${{ github.repository }} | cut -d '/' -f2)" >> $GITHUB_ENV
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push to pyengine
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: pyengine/${{ env.SERVICE }}:${{ env.VERSION }}
+
+      - name: Notice when job fails
+        if: failure()
+        uses: 8398a7/action-slack@v3.2.0
+        with:
+          status: ${{job.status}}
+          fields: repo,workflow,job
+          author_name: Github Action Slack
 
   scan:
-    needs: [versioning, docker]
-    runs-on: ubuntu-20.04
+    needs: [ versioning, docker ]
+    runs-on: ubuntu-latest
     env:
       VERSION: ${{ needs.versioning.outputs.version }}
     steps:
@@ -90,7 +90,7 @@ jobs:
           severity: 'CRITICAL,HIGH'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'
 
@@ -102,7 +102,7 @@ jobs:
           echo "$count"
 
       - name: slack
-        if:  ${{ steps.vulnerabilities.outputs.result_count != 0 }}
+        if: ${{ steps.vulnerabilities.outputs.result_count != 0 }}
         uses: 8398a7/action-slack@v3
         with:
           status: custom

.github/workflows/pull_request_base_check.yaml

@@ -0,0 +1,23 @@
+name: "[Pull Request] Base Check"
+
+on:
+  pull_request_target:
+
+jobs:
+  check-pull-request:
+    name: Check Pull Request
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Check signed commits
+        id: review
+        uses: cloudforet-io/check-pr-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Notify Result
+        if: ${{ steps.review.outputs.signedoff == 'false' }}
+        run: |
+          echo "The review result is ${{ steps.review.outputs.signedoff }}"
+          exit 1

src/VERSION

@@ -1 +1 @@
-1.3.15
+1.3.21

src/cloudforet/cost_analysis/connector/azure_cost_mgmt_connector.py

@@ -24,7 +24,7 @@
 
 _LOGGER = logging.getLogger("spaceone")
 
-_PAGE_SIZE = 7000
+_PAGE_SIZE = 5000
 
 
 def azure_exception_handler(func):
@@ -33,12 +33,23 @@ def wrapper(*args, **kwargs) -> Union[dict, list]:
         return_type = get_type_hints(func).get("return")
         try:
             return func(*args, **kwargs)
+        except ServiceResponseError as error:
+            _print_error_log(error)
+            time.sleep(10)
+            return func(*args, **kwargs)
+
         except ResourceNotFoundError as error:
             _print_error_log(error)
             return _get_empty_value(return_type)
         except HttpResponseError as error:
             if error.status_code in ["404", "412"]:
                 _print_error_log(error)
+            elif error.status_code == "429":
+                _LOGGER.error(f"(RateLimit Error) => {error.message}")
+
+                wait_time = _extract_wait_time_for_retry(error.message)
+                time.sleep(wait_time)
+                return func(*args, **kwargs)
             else:
                 _print_error_log(error)
             return _get_empty_value(return_type)
@@ -49,6 +60,17 @@ def wrapper(*args, **kwargs) -> Union[dict, list]:
     return wrapper
 
 
+def _extract_wait_time_for_retry(message: str) -> int:
+    default_wait_time = 60
+    pattern = r"\d+"
+
+    matches = re.findall(pattern, message)
+
+    if len(matches) > 1:
+        default_wait_time = int(matches[1])
+    return default_wait_time
+
+
 def _get_empty_value(return_type: object) -> Any:
     return_type_name = getattr(return_type, "__name__")
     empty_values = {
@@ -65,8 +87,9 @@ def _get_empty_value(return_type: object) -> Any:
     return empty_values.get(return_type_name, None)
 
 
-def _print_error_log(error):
-    _LOGGER.error(f"(Error) => {error.message} {error}", exc_info=True)
+def _print_error_log(error) -> None:
+    status_code = getattr(error, "status_code", "Unknown")
+    _LOGGER.error(f"(Error) => {status_code} {error.message} {error}", exc_info=True)
 
 
 class AzureCostMgmtConnector(BaseConnector):
@@ -154,11 +177,9 @@ def query_usage_http(
                     BENEFIT_GROUPING + BENEFIT_GROUPING_MCA
                 )
 
-            _LOGGER.debug(f"[query_usage] parameters: {parameters}")
-
             while self.next_link:
                 url = self.next_link
-                headers = self._make_request_headers()
+                headers = self._make_request_headers(secret_data)
 
                 _LOGGER.debug(f"[query_usage] url:{url}, parameters: {parameters}")
                 response = requests.post(url=url, headers=headers, json=parameters)
@@ -298,8 +319,8 @@ def convert_nested_dictionary(self, cloud_svc_object):
 
         return cloud_svc_dict
 
-    def _make_request_headers(self, client_type=None):
-        access_token = self._get_access_token()
+    def _make_request_headers(self, secret_data: dict, client_type=None):
+        access_token = self._get_access_token(secret_data)
         headers = {
             "Authorization": f"Bearer {access_token}",
             "Content-Type": "application/json",
@@ -311,7 +332,7 @@ def _make_request_headers(self, client_type=None):
 
     def _retry_request(self, response, url, headers, json, retry_count, method="post"):
         try:
-            _LOGGER.error(f"{datetime.utcnow()}[INFO] retry_request {response.headers}")
+            _LOGGER.error(f"[INFO] retry_request {response.headers}")
             if retry_count == 0:
                 raise ERROR_UNKNOWN(
                     message=f"[ERROR] retry_request failed {response.json()}"
@@ -366,12 +387,26 @@ def _get_sleep_time(response_headers):
         return sleep_time + 1
 
     @staticmethod
-    def _get_access_token():
+    def _get_access_token(secret_data: dict) -> str:
         try:
-            credential = DefaultAzureCredential(logging_enable=True)
-            scopes = ["https://management.azure.com/.default"]
-            token_info = credential.get_token(*scopes)
-            return token_info.token
+            header = {
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            data = {
+                "client_id": secret_data["client_id"],
+                "client_secret": secret_data["client_secret"],
+                "grant_type": "client_credentials",
+                "resource": "https://management.azure.com",
+                "scope": "https://management.azure.com/.default",
+            }
+
+            response = requests.post(
+                f"https://login.microsoftonline.com/{secret_data['tenant_id']}/oauth2/token",
+                data=data,
+                headers=header,
+            )
+            access_token = response.json().get("access_token")
+            return access_token
         except Exception as e:
             _LOGGER.error(f"[ERROR] _get_access_token :{e}")
             raise ERROR_INVALID_TOKEN(token=e)

src/cloudforet/cost_analysis/main.py

@@ -161,6 +161,7 @@ def cost_get_data(params: dict) -> Generator[dict, None, None]:
 
     params["schema"] = params.pop("schema_name", None)
     params["secret_data"] = __get_secret_data(secret_data, task_options)
+
     is_benefit_job = task_options.get("is_benefit_job", False)
     cost_metric = options.get("cost_metric", "ActualCost")
 
@@ -201,14 +202,15 @@ def __get_secret_data(secret_data: dict, task_options: dict) -> dict:
     if len(secrets) == 1:
         return secrets[0]
 
-    tenant_id = task_options.get(
-        "billing_tenant_id",
-    )
+    billing_tenant_id = task_options["billing_tenant_id"]
 
     for _secret_data in secrets:
-        if _secret_data["tenant_id"] == tenant_id:
-            secret_data = _secret_data
-        elif _secret_data.get("subscription_id") == task_options.get("subscription_id"):
-            secret_data = _secret_data
+        if _secret_data["tenant_id"] == billing_tenant_id:
+            return _secret_data
+
+        elif _secret_data.get("subscription_id") and _secret_data.get(
+            "subscription_id"
+        ) == task_options.get("subscription_id"):
+            return _secret_data
 
-    return secret_data
+    return secrets[0]

src/cloudforet/cost_analysis/manager/cost_manager.py

@@ -70,7 +70,7 @@ def get_data(
         schema: str,
         task_options: dict,
         domain_id: str,
-    ) -> list:
+    ) -> Generator[list, Any, None]:
         self.azure_cm_connector.create_session(options, secret_data, schema)
         self._check_task_options(task_options)
 
@@ -104,6 +104,13 @@ def get_data(
                     scope, parameters
                 )
 
+                if not blobs:
+                    _LOGGER.debug(f"[get_data] blobs: {blobs}")
+                    _LOGGER.info(
+                        f"[get_data] #{idx + 1} {tenant_id} tenant collect skipped, domain_id: {domain_id}"
+                    )
+                    continue
+
                 response_stream = self.azure_cm_connector.get_cost_data(blobs, options)
                 for results in response_stream:
                     yield self._make_cost_data(
@@ -132,7 +139,6 @@ def get_data(
             _LOGGER.info(
                 f"[get_data] all collect is done in {int(end_time - start_time)} seconds"
             )
-        yield []
 
     def _make_cost_data(
         self,
@@ -336,7 +342,7 @@ def _get_additional_info(self, result: dict, options: dict, tenant_id: str = Non
             additional_info["Term"] = term
 
         if azure_additional_info := result.get("additionalinfo"):
-            azure_additional_info: dict = json.loads(azure_additional_info)
+            azure_additional_info = json.loads(azure_additional_info)
             if ri_normalization_ratio := azure_additional_info.get(
                 "RINormalizationRatio"
             ):
@@ -397,11 +403,14 @@ def _make_benefit_cost_data(
         account_agreement_type: str = None,
     ) -> list:
         benefit_costs_data = []
+        total_count = 0
+
         try:
             combined_results = self._combine_rows_and_columns_from_results(
                 results.get("properties").get("rows"),
                 results.get("properties").get("columns"),
             )
+            total_count += len(combined_results)
             for cb_result in combined_results:
                 billed_at = self._set_billed_date(cb_result.get("UsageDate", end))
                 if not billed_at:
@@ -413,13 +422,12 @@ def _make_benefit_cost_data(
         except Exception as e:
             _LOGGER.error(f"[_make_cost_data] make data error: {e}", exc_info=True)
             raise e
-
+        _LOGGER.info(f"[get_benefit_data] total count: {total_count}")
         return benefit_costs_data
 
     def _make_benefit_cost_info(self, result: dict, billed_at: str) -> dict:
         additional_info = {
             "Pricing Model": result.get("PricingModel"),
-            "Frequency": result.get("BillingFrequency"),
             "Benefit Id": result.get("BenefitId"),
             "Benefit Name": result.get("BenefitName"),
             "Reservation Id": result.get("ReservationId"),

src/cloudforet/cost_analysis/manager/job_manager.py

@@ -12,7 +12,7 @@
 
 _LOGGER = logging.getLogger("spaceone")
 
-_TASK_LIST_SIZE = 5
+_TASK_LIST_SIZE = 4
 
 
 class JobManager(BaseManager):