--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -112,21 +112,18 @@ def rewrite_when_to_optimization(config,
     for job in jobs:
         when = job.pop('when', {})
         if not when:
             yield job
             continue
 
         files_changed = when.get('files-changed')
 
-        # implicitly add config directories affecting task
+        # implicitly add task config directory.
         files_changed.append('{}/**'.format(config.path))
-        if 'in-tree' in job.get('worker', {}).get('docker-image', {}):
-            files_changed.append('taskcluster/docker/{}/**'.format(
-                job['worker']['docker-image']['in-tree']))
 
         # "only when files changed" implies "skip if files have not changed"
         job['optimization'] = {'skip-unless-changed': files_changed}
 
         assert 'when' not in job
         yield job