diff --git a/builder/internals.go b/builder/internals.go
index e554fd302..f6b929f2b 100644
--- a/builder/internals.go
+++ b/builder/internals.go
@@ -187,8 +187,8 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
 	if err != nil {
 		return err
 	}
-	// If we do not have at least one hash, never use the cache
-	if hit && b.UtilizeCache {
+
+	if hit {
 		return nil
 	}