fix: dlp samples for grpc (GoogleCloudPlatform#1123)

Author: bshaffer

.kokoro/secrets-example.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 
-# This file contains the necessary environment variables for the kokoro 
+# This file contains the necessary environment variables for the kokoro
 # tests. Contact the repository owners if you need access to view or modify
 # the variables.
-# 
+#
 # Run the following gcloud command to decrypt secrets.sh.enc as follows:
 #
 # gcloud kms decrypt --location=global --keyring=ci --key=ci \
 #   --ciphertext-file=.kokoro/secrets.sh.enc \
 #   --plaintext-file=.kokoro/secrets.sh
 #
 # Then run `source .kokoro/secrets.sh`
-# 
-# To modify the file, edit .kokoro/secrets.sh then use the following gcloud 
+#
+# To modify the file, edit .kokoro/secrets.sh then use the following gcloud
 # command to encrypt it with the changes:
 #
 # gcloud kms encrypt --location=global --keyring=ci --key=ci \
@@ -66,8 +66,6 @@ export CLOUD_DATASTORE_NAMESPACE=
 export DATASTORE_EVENTUALLY_CONSISTENT_RETRY_COUNT=
 
 # DLP
-export DLP_TOPIC=dlp-tests
-export DLP_SUBSCRIPTION=dlp-tests
 export DLP_DEID_WRAPPED_KEY=
 export DLP_DEID_KEY_NAME=projects/$GOOGLE_PROJECT_ID/locations/global/keyRings/ci/cryptoKeys/ci
 

.kokoro/secrets.sh.enc

@@ -34,7 +34,6 @@
 /**
  * Computes risk metrics of a column of data in a Google BigQuery table.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
 use Google\Cloud\Dlp\V2\BigQueryTable;
@@ -94,7 +93,7 @@
     ->setActions([$action]);
 
 // Submit request
-$parent = "projects/$callingProjectId/locations/global";;
+$parent = "projects/$callingProjectId/locations/global";
 $job = $dlp->createDlpJob($parent, [
     'riskJob' => $riskJob
 ]);
@@ -103,9 +102,10 @@
 $subscription = $topic->subscription($subscriptionId);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -114,11 +114,13 @@
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -151,6 +153,9 @@
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         printf('Unexpected job state.');
 }

dlp/src/categorical_stats.php

@@ -44,7 +44,7 @@
 $dlp = new DlpServiceClient();
 
 // Run template deletion request
-$templateName = $dlp->projectInspectTemplateName($callingProjectId, $templateId);
+$templateName = "projects/$callingProjectId/locations/global/inspectTemplates/$templateId";
 $dlp->deleteInspectTemplate($templateName);
 
 // Print results

dlp/src/delete_inspect_template.php

@@ -45,7 +45,7 @@
 
 // Run request
 // The Parent project ID is automatically extracted from this parameter
-$triggerName = $dlp->projectJobTriggerName($callingProjectId, $triggerId);
+$triggerName = "projects/$callingProjectId/locations/global/jobTriggers/$triggerId";
 $response = $dlp->deleteJobTrigger($triggerName);
 
 // Print the results

dlp/src/delete_trigger.php

@@ -35,7 +35,6 @@
 /**
  * Inspect a BigQuery table , using Pub/Sub for job status notifications.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\BigQueryOptions;
 use Google\Cloud\Dlp\V2\InfoType;
@@ -120,9 +119,10 @@
 ]);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -131,11 +131,13 @@
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         printf('Unexpected job state. Most likely, the job is either running or has not yet started.');
 }

dlp/src/inspect_bigquery.php

@@ -35,7 +35,6 @@
 /**
  * Inspect Datastore, using Pub/Sub for job status notifications.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\DatastoreOptions;
 use Google\Cloud\Dlp\V2\InfoType;
@@ -124,9 +123,10 @@
 ]);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -135,11 +135,13 @@
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         print('Unexpected job state.');
 }

dlp/src/inspect_datastore.php

@@ -35,7 +35,6 @@
 /**
  * Inspect a file stored on Google Cloud Storage , using Pub/Sub for job status notifications.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\CloudStorageOptions;
 use Google\Cloud\Dlp\V2\CloudStorageOptions\FileSet;
@@ -120,9 +119,10 @@
 ]);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -131,11 +131,13 @@
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -157,6 +159,9 @@
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         print('Unexpected job state. Most likely, the job is either running or has not yet started.');
 }

dlp/src/inspect_gcs.php

@@ -34,7 +34,6 @@
 /**
  * Computes the k-anonymity of a column set in a Google BigQuery table.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
 use Google\Cloud\Dlp\V2\BigQueryTable;
@@ -107,9 +106,10 @@ function ($id) {
 ]);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -118,11 +118,13 @@ function ($id) {
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -161,6 +163,9 @@ function ($id) {
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         printf('Unexpected job state. Most likely, the job is either running or has not yet started.');
 }

dlp/src/k_anonymity.php

@@ -29,6 +29,7 @@
     return print("Usage: php k_map.php CALLING_PROJECT DATA_PROJECT TOPIC SUBSCRIPTION DATASET TABLE REGION_CODE QUASI_ID_NAMES INFO_TYPES\n");
 }
 list($_, $callingProjectId, $dataProjectId, $topicId, $subscriptionId, $datasetId, $tableId, $regionCode, $quasiIdNames, $infoTypes) = $argv;
+
 // Convert comma-separated lists to arrays
 $quasiIdNames = explode(',', $quasiIdNames);
 $infoTypes = explode(',', $infoTypes);
@@ -37,7 +38,6 @@
 /**
  * Computes the k-map risk estimation of a column set in a Google BigQuery table.
  */
-use Google\Cloud\Core\ExponentialBackoff;
 use Google\Cloud\Dlp\V2\DlpServiceClient;
 use Google\Cloud\Dlp\V2\InfoType;
 use Google\Cloud\Dlp\V2\RiskAnalysisJobConfig;
@@ -128,9 +128,10 @@
 ]);
 
 // Poll Pub/Sub using exponential backoff until job finishes
-$backoff = new ExponentialBackoff(30);
-$backoff->execute(function () use ($subscription, $dlp, &$job) {
-    printf('Waiting for job to complete' . PHP_EOL);
+// Consider using an asynchronous execution model such as Cloud Functions
+$attempt = 1;
+$startTime = time();
+do {
     foreach ($subscription->pull() as $message) {
         if (isset($message->attributes()['DlpJobName']) &&
             $message->attributes()['DlpJobName'] === $job->getName()) {
@@ -139,11 +140,13 @@
             do {
                 $job = $dlp->getDlpJob($job->getName());
             } while ($job->getState() == JobState::RUNNING);
-            return true;
+            break 2; // break from parent do while
         }
     }
-    throw new Exception('Job has not yet completed');
-});
+    printf('Waiting for job to complete' . PHP_EOL);
+    // Exponential backoff with max delay of 60 seconds
+    sleep(min(60, pow(2, ++$attempt)));
+} while (time() - $startTime < 600); // 10 minute timeout
 
 // Print finding counts
 printf('Job %s status: %s' . PHP_EOL, $job->getName(), JobState::name($job->getState()));
@@ -183,6 +186,9 @@
             var_dump($error->getDetails());
         }
         break;
+    case JobState::PENDING:
+        printf('Job has not completed. Consider a longer timeout or an asynchronous execution model' . PHP_EOL);
+        break;
     default:
         print('Unexpected job state. Most likely, the job is either running or has not yet started.');
 }